



<!doctype html>
<html lang="en" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      <meta http-equiv="x-ua-compatible" content="ie=edge">
      
        <meta name="description" content="Deep learning toolbox">
      
      
        <link rel="canonical" href="https://uber.github.io/ludwig/user_guide/">
      
      
        <meta name="author" content="Piero Molino">
      
      
        <meta name="lang:clipboard.copy" content="Copy to clipboard">
      
        <meta name="lang:clipboard.copied" content="Copied to clipboard">
      
        <meta name="lang:search.language" content="en">
      
        <meta name="lang:search.pipeline.stopwords" content="True">
      
        <meta name="lang:search.pipeline.trimmer" content="True">
      
        <meta name="lang:search.result.none" content="No matching documents">
      
        <meta name="lang:search.result.one" content="1 matching document">
      
        <meta name="lang:search.result.other" content="# matching documents">
      
        <meta name="lang:search.tokenizer" content="[\s\-]+">
      
      <link rel="shortcut icon" href="../images/ludwig_logo.png">
      <meta content="mkdocs-1.0.4, mkdocs-material-3.3.0" name="generator">
    
    
      
        <title>User Guide - Ludwig</title>
      
    
    
      <link rel="stylesheet" href="../assets/stylesheets/application.572ca0f0.css">
      
        <link rel="stylesheet" href="../assets/stylesheets/application-palette.22915126.css">
      
      
        
        
        <meta name="theme-color" content="#757575">


      <script src="../assets/javascripts/modernizr.962652e9.js"></script>
    
    
      
        <link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono">
        <style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
      
    
    <link rel="stylesheet" href="../assets/fonts/material-icons.css">
    
    
      <link rel="stylesheet" href="../stylesheets/extra.css">
    
      <link rel="stylesheet" href="../stylesheets/monokai.css">


  </head>
  
    
    
    <body dir="ltr" data-md-color-primary="grey" data-md-color-accent="grey">
  
    <svg class="md-svg">
      <defs>
        
        
          <svg xmlns="http://www.w3.org/2000/svg" width="416" height="448"
    viewBox="0 0 416 448" id="__github">
  <path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19-18.125
        8.5-18.125-8.5-10.75-19-3.125-20.5 3.125-20.5 10.75-19 18.125-8.5
        18.125 8.5 10.75 19 3.125 20.5zM320 304q0 10-3.125 20.5t-10.75
        19-18.125 8.5-18.125-8.5-10.75-19-3.125-20.5 3.125-20.5 10.75-19
        18.125-8.5 18.125 8.5 10.75 19 3.125 20.5zM360
        304q0-30-17.25-51t-46.75-21q-10.25 0-48.75 5.25-17.75 2.75-39.25
        2.75t-39.25-2.75q-38-5.25-48.75-5.25-29.5 0-46.75 21t-17.25 51q0 22 8
        38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0
        37.25-1.75t35-7.375 30.5-15 20.25-25.75 8-38.375zM416 260q0 51.75-15.25
        82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5-41.75
        1.125q-19.5 0-35.5-0.75t-36.875-3.125-38.125-7.5-34.25-12.875-30.25-20.25-21.5-28.75q-15.5-30.75-15.5-82.75
        0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25
        30.875q36.75-8.75 77.25-8.75 37 0 70 8 26.25-20.5
        46.75-30.25t47.25-9.75q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34
        99.5z" />
</svg>
        
      </defs>
    </svg>
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
    
      <a href="#command-line-interface" tabindex="1" class="md-skip">
        Skip to content
      </a>
    
    
      <header class="md-header" data-md-component="header">
    <nav class="md-header-nav md-grid">
        <div class="md-flex">
            <div class="md-flex__cell md-flex__cell--shrink">
                <a class="md-header-nav__button md-logo"
                   href="https://uber.github.io/ludwig/" title="Ludwig">
                    <img src="../images/ludwig_logo.svg" style="height:2rem;">
                </a>
            </div>
            <div class="md-flex__cell md-flex__cell--shrink">
                <label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
            </div>
            <div class="md-flex__cell md-flex__cell--stretch">
                <div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
                    
                    <span class="md-header-nav__topic">
                    User Guide
                    </span>
                    
                </div>
            </div>
            <div class="md-flex__cell md-flex__cell--shrink">
                
                
                <label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
                
<div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
      <label class="md-icon md-search__icon" for="__search"></label>
      <button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
        &#xE5CD;
      </button>
    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" data-md-scrollfix>
        <div class="md-search-result" data-md-component="result">
          <div class="md-search-result__meta">
            Type to start searching
          </div>
          <ol class="md-search-result__list"></ol>
        </div>
      </div>
    </div>
  </div>
</div>
                
                
            </div>
            
            <div class="md-flex__cell md-flex__cell--shrink">
                <div class="md-header-nav__source">
                    


  


  <a href="https://github.com/uber/ludwig/" title="Go to repository" class="md-source" data-md-source="github">
    
      <div class="md-source__icon">
        <svg viewBox="0 0 24 24" width="24" height="24">
          <use xlink:href="#__github" width="24" height="24"></use>
        </svg>
      </div>
    
    <div class="md-source__repository">
      uber/ludwig
    </div>
  </a>

                </div>
            </div>
            
        </div>
    </nav>
</header>
    
    <div class="md-container">
      
        
      
      
      <main class="md-main">
        <div class="md-main__inner md-grid" data-md-component="container">
          
            
              <div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    <nav class="md-nav md-nav--primary" data-md-level="0">
    <label class="md-nav__title md-nav__title--site" for="__drawer">
        <a class="md-nav__button md-logo" href="https://uber.github.io/ludwig/"
           title="Ludwig">
            <img src="../images/ludwig_logo.svg" style="width: 20.8rem">
        </a>
    </label>
    
    <div class="md-nav__source">
        


  


  <a href="https://github.com/uber/ludwig/" title="Go to repository" class="md-source" data-md-source="github">
    
      <div class="md-source__icon">
        <svg viewBox="0 0 24 24" width="24" height="24">
          <use xlink:href="#__github" width="24" height="24"></use>
        </svg>
      </div>
    
    <div class="md-source__repository">
      uber/ludwig
    </div>
  </a>

    </div>
    
    <ul class="md-nav__list" data-md-scrollfix>
        
        
        
        


  <li class="md-nav__item">
    <a href=".." title="About" class="md-nav__link">
      About
    </a>
  </li>

        
        
        
        


  <li class="md-nav__item">
    <a href="../getting_started/" title="Getting Started" class="md-nav__link">
      Getting Started
    </a>
  </li>

        
        
        
        


  <li class="md-nav__item">
    <a href="../examples/" title="Examples" class="md-nav__link">
      Examples
    </a>
  </li>

        
        
        
        

  


  <li class="md-nav__item md-nav__item--active">
    
    <input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
    
    
      <label class="md-nav__link md-nav__link--active" for="__toc">
        User Guide
      </label>
    
    <a href="./" title="User Guide" class="md-nav__link md-nav__link--active">
      User Guide
    </a>
    
      
<nav class="md-nav md-nav--secondary">
  
  
  
    <label class="md-nav__title" for="__toc">Table of contents</label>
    <ul class="md-nav__list" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#command-line-interface" title="Command Line Interface" class="md-nav__link">
    Command Line Interface
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#train" title="train" class="md-nav__link">
    train
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#predict" title="predict" class="md-nav__link">
    predict
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#experiment" title="experiment" class="md-nav__link">
    experiment
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#visualize" title="visualize" class="md-nav__link">
    visualize
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#collect_weights" title="collect_weights" class="md-nav__link">
    collect_weights
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#collect_activations" title="collect_activations" class="md-nav__link">
    collect_activations
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#data-preprocessing" title="Data Preprocessing" class="md-nav__link">
    Data Preprocessing
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#csv-format" title="CSV Format" class="md-nav__link">
    CSV Format
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#data-postprocessing" title="Data Postprocessing" class="md-nav__link">
    Data Postprocessing
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#model-definition" title="Model Definition" class="md-nav__link">
    Model Definition
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#input-features" title="Input features" class="md-nav__link">
    Input features
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#combiner" title="Combiner" class="md-nav__link">
    Combiner
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#output-features" title="Output Features" class="md-nav__link">
    Output Features
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#output-features-dependencies" title="Output Features Dependencies" class="md-nav__link">
    Output Features Dependencies
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#training" title="Training" class="md-nav__link">
    Training
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#preprocessing" title="Preprocessing" class="md-nav__link">
    Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#binary-features" title="Binary Features" class="md-nav__link">
    Binary Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#binary-features-preprocessing" title="Binary Features Preprocessing" class="md-nav__link">
    Binary Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#binary-input-features-and-encoders" title="Binary Input Features and Encoders" class="md-nav__link">
    Binary Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#binary-output-features-and-decoders" title="Binary Output Features and Decoders" class="md-nav__link">
    Binary Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#binary-features-measures" title="Binary Features Measures" class="md-nav__link">
    Binary Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#numerical-features" title="Numerical Features" class="md-nav__link">
    Numerical Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#numerical-features-preprocessing" title="Numerical Features Preprocessing" class="md-nav__link">
    Numerical Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#numerical-input-features-and-encoders" title="Numerical Input Features and Encoders" class="md-nav__link">
    Numerical Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#numerical-output-features-and-decoders" title="Numerical Output Features and Decoders" class="md-nav__link">
    Numerical Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#numerical-features-measures" title="Numerical Features Measures" class="md-nav__link">
    Numerical Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#category-features" title="Category Features" class="md-nav__link">
    Category Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#category-features-preprocessing" title="Category Features Preprocessing" class="md-nav__link">
    Category Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#category-input-features-and-encoders" title="Category Input Features and Encoders" class="md-nav__link">
    Category Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#category-output-features-and-decoders" title="Category Output Features and Decoders" class="md-nav__link">
    Category Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#category-features-measures" title="Category Features Measures" class="md-nav__link">
    Category Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#set-features" title="Set Features" class="md-nav__link">
    Set Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#set-features-preprocessing" title="Set Features Preprocessing" class="md-nav__link">
    Set Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#set-input-features-and-encoders" title="Set Input Features and Encoders" class="md-nav__link">
    Set Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#set-output-features-and-decoders" title="Set Output Features and Decoders" class="md-nav__link">
    Set Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#set-features-measures" title="Set Features Measures" class="md-nav__link">
    Set Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#bag-features" title="Bag Features" class="md-nav__link">
    Bag Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#bag-features-preprocessing" title="Bag Features Preprocessing" class="md-nav__link">
    Bag Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#bag-input-features-and-encoders" title="Bag Input Features and Encoders" class="md-nav__link">
    Bag Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#bag-output-features-and-decoders" title="Bag Output Features and Decoders" class="md-nav__link">
    Bag Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#bag-features-measures" title="Bag Features Measures" class="md-nav__link">
    Bag Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-features" title="Sequence Features" class="md-nav__link">
    Sequence Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#sequence-features-preprocessing" title="Sequence Features Preprocessing" class="md-nav__link">
    Sequence Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-input-features-and-encoders" title="Sequence Input Features and Encoders" class="md-nav__link">
    Sequence Input Features and Encoders
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#embed-encoder" title="Embed Encoder" class="md-nav__link">
    Embed Encoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#parallel-cnn-encoder" title="Parallel CNN Encoder" class="md-nav__link">
    Parallel CNN Encoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#stacked-cnn-encoder" title="Stacked CNN Encoder" class="md-nav__link">
    Stacked CNN Encoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#stacked-parallel-cnn-encoder" title="Stacked Parallel CNN Encoder" class="md-nav__link">
    Stacked Parallel CNN Encoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#rnn-encoder" title="RNN Encoder" class="md-nav__link">
    RNN Encoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#cnn-rnn-encoder" title="CNN RNN Encoder" class="md-nav__link">
    CNN RNN Encoder
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-output-features-and-decoders" title="Sequence Output Features and Decoders" class="md-nav__link">
    Sequence Output Features and Decoders
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#tagger-decoder" title="Tagger Decoder" class="md-nav__link">
    Tagger Decoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#generator-decoder" title="Generator Decoder" class="md-nav__link">
    Generator Decoder
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-features-measures" title="Sequence Features Measures" class="md-nav__link">
    Sequence Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#text-features" title="Text Features" class="md-nav__link">
    Text Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#text-features-preprocessing" title="Text Features Preprocessing" class="md-nav__link">
    Text Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#text-input-features-and-encoders" title="Text Input Features and Encoders" class="md-nav__link">
    Text Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#text-output-features-and-decoders" title="Text Output Features and Decoders" class="md-nav__link">
    Text Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#text-features-measures" title="Text Features Measures" class="md-nav__link">
    Text Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#time-series-features" title="Time Series Features" class="md-nav__link">
    Time Series Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#time-series-features-preprocessing" title="Time Series Features Preprocessing" class="md-nav__link">
    Time Series Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#time-series-input-features-and-encoders" title="Time Series Input Features and Encoders" class="md-nav__link">
    Time Series Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#time-series-output-features-and-decoders" title="Time Series Output Features and Decoders" class="md-nav__link">
    Time Series Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#time-series-features-measures" title="Time Series Features Measures" class="md-nav__link">
    Time Series Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#image-features" title="Image Features" class="md-nav__link">
    Image Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#image-features-preprocessing" title="Image Features Preprocessing" class="md-nav__link">
    Image Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#image-input-features-and-encoders" title="Image Input Features and Encoders" class="md-nav__link">
    Image Input Features and Encoders
  </a>

              <nav class="md-nav">
                  <ul class="md-nav__list">

                      <li class="md-nav__item">
                          <a class="md-nav__link" href="#convolutional-stack-encoder"
                             title="Convolutional Stack Encoder">
                              Convolutional Stack Encoder
                          </a>

                      </li>

                      <li class="md-nav__item">
                          <a class="md-nav__link" href="#resnet-encoder" title="ResNet Encoder">
                              ResNet Encoder
                          </a>

                      </li>

                  </ul>
              </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#image-output-features-and-decoders" title="Image Output Features and Decoders" class="md-nav__link">
    Image Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#image-features-measures" title="Image Features Measures" class="md-nav__link">
    Image Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#combiners" title="Combiners" class="md-nav__link">
    Combiners
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#concat-combiner" title="Concat Combiner" class="md-nav__link">
    Concat Combiner
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-concat-combiner" title="Sequence Concat Combiner" class="md-nav__link">
    Sequence Concat Combiner
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-combiner" title="Sequence Combiner" class="md-nav__link">
    Sequence Combiner
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#distributed-training" title="Distributed Training" class="md-nav__link">
    Distributed Training
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#programatic-api" title="Programatic API" class="md-nav__link">
    Programatic API
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#training-a-model" title="Training a Model" class="md-nav__link">
    Training a Model
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#loading-a-pre-trained-model" title="Loading a Pre-trained Model" class="md-nav__link">
    Loading a Pre-trained Model
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#predicting" title="Predicting" class="md-nav__link">
    Predicting
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#visualizations" title="Visualizations" class="md-nav__link">
    Visualizations
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#learning-curves" title="Learning Curves" class="md-nav__link">
    Learning Curves
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#learning_curves" title="learning_curves" class="md-nav__link">
    learning_curves
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confusion-matrix" title="Confusion Matrix" class="md-nav__link">
    Confusion Matrix
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#confusion_matrix" title="confusion_matrix" class="md-nav__link">
    confusion_matrix
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare-performance" title="Compare Performance" class="md-nav__link">
    Compare Performance
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#compare_performance" title="compare_performance" class="md-nav__link">
    compare_performance
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_performance_from_prob" title="compare_classifiers_performance_from_prob" class="md-nav__link">
    compare_classifiers_performance_from_prob
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_performance_from_pred" title="compare_classifiers_performance_from_pred" class="md-nav__link">
    compare_classifiers_performance_from_pred
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_performance_subset" title="compare_classifiers_performance_subset" class="md-nav__link">
    compare_classifiers_performance_subset
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_performance_changing_k" title="compare_classifiers_performance_changing_k" class="md-nav__link">
    compare_classifiers_performance_changing_k
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_multiclass_multimetric" title="compare_classifiers_multiclass_multimetric" class="md-nav__link">
    compare_classifiers_multiclass_multimetric
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare-classifier-predictions" title="Compare Classifier Predictions" class="md-nav__link">
    Compare Classifier Predictions
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_predictions" title="compare_classifiers_predictions" class="md-nav__link">
    compare_classifiers_predictions
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_predictions_distribution" title="compare_classifiers_predictions_distribution" class="md-nav__link">
    compare_classifiers_predictions_distribution
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding" title="Confidence_Thresholding" class="md-nav__link">
    Confidence_Thresholding
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_1" title="confidence_thresholding" class="md-nav__link">
    confidence_thresholding
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_data_vs_acc" title="confidence_thresholding_data_vs_acc" class="md-nav__link">
    confidence_thresholding_data_vs_acc
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_data_vs_acc_subset" title="confidence_thresholding_data_vs_acc_subset" class="md-nav__link">
    confidence_thresholding_data_vs_acc_subset
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_data_vs_acc_subset_per_class" title="confidence_thresholding_data_vs_acc_subset_per_class" class="md-nav__link">
    confidence_thresholding_data_vs_acc_subset_per_class
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_2thresholds_2d" title="confidence_thresholding_2thresholds_2d" class="md-nav__link">
    confidence_thresholding_2thresholds_2d
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_2thresholds_3d" title="confidence_thresholding_2thresholds_3d" class="md-nav__link">
    confidence_thresholding_2thresholds_3d
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#binary-threshold-vs-metric" title="Binary Threshold vs. Metric" class="md-nav__link">
    Binary Threshold vs. Metric
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#binary_threshold_vs_metric" title="binary_threshold_vs_metric" class="md-nav__link">
    binary_threshold_vs_metric
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#roc-curves" title="ROC Curves" class="md-nav__link">
    ROC Curves
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#roc_curves" title="roc_curves" class="md-nav__link">
    roc_curves
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#roc_curves_from_test_stats" title="roc_curves_from_test_stats" class="md-nav__link">
    roc_curves_from_test_stats
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#calibration-plot" title="Calibration Plot" class="md-nav__link">
    Calibration Plot
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#calibration_1_vs_all" title="calibration_1_vs_all" class="md-nav__link">
    calibration_1_vs_all
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#calibration_multiclass" title="calibration_multiclass" class="md-nav__link">
    calibration_multiclass
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#class-frequency-vs-f1-score" title="Class Frequency vs. F1 score" class="md-nav__link">
    Class Frequency vs. F1 score
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#frequency_vs_f1" title="frequency_vs_f1" class="md-nav__link">
    frequency_vs_f1
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
      
      
      
      
    </ul>
  
</nav>
    
  </li>

        
        
        
        


  <li class="md-nav__item">
    <a href="../developer_guide/" title="Developer Guide" class="md-nav__link">
      Developer Guide
    </a>
  </li>

        
        
        
        


  <li class="md-nav__item">
    <a href="../api/" title="API" class="md-nav__link">
      API
    </a>
  </li>

        
        
        
        


  <li class="md-nav__item">
    <a href="../faq/" title="FAQ" class="md-nav__link">
      FAQ
    </a>
  </li>

        
    </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
              <div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    
<nav class="md-nav md-nav--secondary">
  
  
  
    <label class="md-nav__title" for="__toc">Table of contents</label>
    <ul class="md-nav__list" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#command-line-interface" title="Command Line Interface" class="md-nav__link">
    Command Line Interface
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#train" title="train" class="md-nav__link">
    train
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#predict" title="predict" class="md-nav__link">
    predict
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#experiment" title="experiment" class="md-nav__link">
    experiment
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#visualize" title="visualize" class="md-nav__link">
    visualize
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#collect_weights" title="collect_weights" class="md-nav__link">
    collect_weights
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#collect_activations" title="collect_activations" class="md-nav__link">
    collect_activations
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#data-preprocessing" title="Data Preprocessing" class="md-nav__link">
    Data Preprocessing
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#csv-format" title="CSV Format" class="md-nav__link">
    CSV Format
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#data-postprocessing" title="Data Postprocessing" class="md-nav__link">
    Data Postprocessing
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#model-definition" title="Model Definition" class="md-nav__link">
    Model Definition
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#input-features" title="Input features" class="md-nav__link">
    Input features
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#combiner" title="Combiner" class="md-nav__link">
    Combiner
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#output-features" title="Output Features" class="md-nav__link">
    Output Features
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#output-features-dependencies" title="Output Features Dependencies" class="md-nav__link">
    Output Features Dependencies
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#training" title="Training" class="md-nav__link">
    Training
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#preprocessing" title="Preprocessing" class="md-nav__link">
    Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#binary-features" title="Binary Features" class="md-nav__link">
    Binary Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#binary-features-preprocessing" title="Binary Features Preprocessing" class="md-nav__link">
    Binary Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#binary-input-features-and-encoders" title="Binary Input Features and Encoders" class="md-nav__link">
    Binary Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#binary-output-features-and-decoders" title="Binary Output Features and Decoders" class="md-nav__link">
    Binary Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#binary-features-measures" title="Binary Features Measures" class="md-nav__link">
    Binary Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#numerical-features" title="Numerical Features" class="md-nav__link">
    Numerical Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#numerical-features-preprocessing" title="Numerical Features Preprocessing" class="md-nav__link">
    Numerical Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#numerical-input-features-and-encoders" title="Numerical Input Features and Encoders" class="md-nav__link">
    Numerical Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#numerical-output-features-and-decoders" title="Numerical Output Features and Decoders" class="md-nav__link">
    Numerical Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#numerical-features-measures" title="Numerical Features Measures" class="md-nav__link">
    Numerical Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#category-features" title="Category Features" class="md-nav__link">
    Category Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#category-features-preprocessing" title="Category Features Preprocessing" class="md-nav__link">
    Category Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#category-input-features-and-encoders" title="Category Input Features and Encoders" class="md-nav__link">
    Category Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#category-output-features-and-decoders" title="Category Output Features and Decoders" class="md-nav__link">
    Category Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#category-features-measures" title="Category Features Measures" class="md-nav__link">
    Category Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#set-features" title="Set Features" class="md-nav__link">
    Set Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#set-features-preprocessing" title="Set Features Preprocessing" class="md-nav__link">
    Set Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#set-input-features-and-encoders" title="Set Input Features and Encoders" class="md-nav__link">
    Set Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#set-output-features-and-decoders" title="Set Output Features and Decoders" class="md-nav__link">
    Set Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#set-features-measures" title="Set Features Measures" class="md-nav__link">
    Set Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#bag-features" title="Bag Features" class="md-nav__link">
    Bag Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#bag-features-preprocessing" title="Bag Features Preprocessing" class="md-nav__link">
    Bag Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#bag-input-features-and-encoders" title="Bag Input Features and Encoders" class="md-nav__link">
    Bag Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#bag-output-features-and-decoders" title="Bag Output Features and Decoders" class="md-nav__link">
    Bag Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#bag-features-measures" title="Bag Features Measures" class="md-nav__link">
    Bag Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-features" title="Sequence Features" class="md-nav__link">
    Sequence Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#sequence-features-preprocessing" title="Sequence Features Preprocessing" class="md-nav__link">
    Sequence Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-input-features-and-encoders" title="Sequence Input Features and Encoders" class="md-nav__link">
    Sequence Input Features and Encoders
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#embed-encoder" title="Embed Encoder" class="md-nav__link">
    Embed Encoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#parallel-cnn-encoder" title="Parallel CNN Encoder" class="md-nav__link">
    Parallel CNN Encoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#stacked-cnn-encoder" title="Stacked CNN Encoder" class="md-nav__link">
    Stacked CNN Encoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#stacked-parallel-cnn-encoder" title="Stacked Parallel CNN Encoder" class="md-nav__link">
    Stacked Parallel CNN Encoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#rnn-encoder" title="RNN Encoder" class="md-nav__link">
    RNN Encoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#cnn-rnn-encoder" title="CNN RNN Encoder" class="md-nav__link">
    CNN RNN Encoder
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-output-features-and-decoders" title="Sequence Output Features and Decoders" class="md-nav__link">
    Sequence Output Features and Decoders
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#tagger-decoder" title="Tagger Decoder" class="md-nav__link">
    Tagger Decoder
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#generator-decoder" title="Generator Decoder" class="md-nav__link">
    Generator Decoder
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-features-measures" title="Sequence Features Measures" class="md-nav__link">
    Sequence Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#text-features" title="Text Features" class="md-nav__link">
    Text Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#text-features-preprocessing" title="Text Features Preprocessing" class="md-nav__link">
    Text Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#text-input-features-and-encoders" title="Text Input Features and Encoders" class="md-nav__link">
    Text Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#text-output-features-and-decoders" title="Text Output Features and Decoders" class="md-nav__link">
    Text Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#text-features-measures" title="Text Features Measures" class="md-nav__link">
    Text Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#time-series-features" title="Time Series Features" class="md-nav__link">
    Time Series Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#time-series-features-preprocessing" title="Time Series Features Preprocessing" class="md-nav__link">
    Time Series Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#time-series-input-features-and-encoders" title="Time Series Input Features and Encoders" class="md-nav__link">
    Time Series Input Features and Encoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#time-series-output-features-and-decoders" title="Time Series Output Features and Decoders" class="md-nav__link">
    Time Series Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#time-series-features-measures" title="Time Series Features Measures" class="md-nav__link">
    Time Series Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#image-features" title="Image Features" class="md-nav__link">
    Image Features
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#image-features-preprocessing" title="Image Features Preprocessing" class="md-nav__link">
    Image Features Preprocessing
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#image-input-features-and-encoders" title="Image Input Features and Encoders" class="md-nav__link">
    Image Input Features and Encoders
  </a>

              <nav class="md-nav">
                  <ul class="md-nav__list">

                      <li class="md-nav__item">
                          <a class="md-nav__link" href="#convolutional-stack-encoder"
                             title="Convolutional Stack Encoder">
                              Convolutional Stack Encoder
                          </a>

                      </li>

                      <li class="md-nav__item">
                          <a class="md-nav__link" href="#resnet-encoder" title="ResNet Encoder">
                              ResNet Encoder
                          </a>

                      </li>

                  </ul>
              </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#image-output-features-and-decoders" title="Image Output Features and Decoders" class="md-nav__link">
    Image Output Features and Decoders
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#image-features-measures" title="Image Features Measures" class="md-nav__link">
    Image Features Measures
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#combiners" title="Combiners" class="md-nav__link">
    Combiners
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#concat-combiner" title="Concat Combiner" class="md-nav__link">
    Concat Combiner
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-concat-combiner" title="Sequence Concat Combiner" class="md-nav__link">
    Sequence Concat Combiner
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#sequence-combiner" title="Sequence Combiner" class="md-nav__link">
    Sequence Combiner
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#distributed-training" title="Distributed Training" class="md-nav__link">
    Distributed Training
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#programatic-api" title="Programatic API" class="md-nav__link">
    Programatic API
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#training-a-model" title="Training a Model" class="md-nav__link">
    Training a Model
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#loading-a-pre-trained-model" title="Loading a Pre-trained Model" class="md-nav__link">
    Loading a Pre-trained Model
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#predicting" title="Predicting" class="md-nav__link">
    Predicting
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#visualizations" title="Visualizations" class="md-nav__link">
    Visualizations
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#learning-curves" title="Learning Curves" class="md-nav__link">
    Learning Curves
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#learning_curves" title="learning_curves" class="md-nav__link">
    learning_curves
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confusion-matrix" title="Confusion Matrix" class="md-nav__link">
    Confusion Matrix
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#confusion_matrix" title="confusion_matrix" class="md-nav__link">
    confusion_matrix
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare-performance" title="Compare Performance" class="md-nav__link">
    Compare Performance
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#compare_performance" title="compare_performance" class="md-nav__link">
    compare_performance
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_performance_from_prob" title="compare_classifiers_performance_from_prob" class="md-nav__link">
    compare_classifiers_performance_from_prob
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_performance_from_pred" title="compare_classifiers_performance_from_pred" class="md-nav__link">
    compare_classifiers_performance_from_pred
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_performance_subset" title="compare_classifiers_performance_subset" class="md-nav__link">
    compare_classifiers_performance_subset
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_performance_changing_k" title="compare_classifiers_performance_changing_k" class="md-nav__link">
    compare_classifiers_performance_changing_k
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_multiclass_multimetric" title="compare_classifiers_multiclass_multimetric" class="md-nav__link">
    compare_classifiers_multiclass_multimetric
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare-classifier-predictions" title="Compare Classifier Predictions" class="md-nav__link">
    Compare Classifier Predictions
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_predictions" title="compare_classifiers_predictions" class="md-nav__link">
    compare_classifiers_predictions
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#compare_classifiers_predictions_distribution" title="compare_classifiers_predictions_distribution" class="md-nav__link">
    compare_classifiers_predictions_distribution
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding" title="Confidence_Thresholding" class="md-nav__link">
    Confidence_Thresholding
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_1" title="confidence_thresholding" class="md-nav__link">
    confidence_thresholding
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_data_vs_acc" title="confidence_thresholding_data_vs_acc" class="md-nav__link">
    confidence_thresholding_data_vs_acc
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_data_vs_acc_subset" title="confidence_thresholding_data_vs_acc_subset" class="md-nav__link">
    confidence_thresholding_data_vs_acc_subset
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_data_vs_acc_subset_per_class" title="confidence_thresholding_data_vs_acc_subset_per_class" class="md-nav__link">
    confidence_thresholding_data_vs_acc_subset_per_class
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_2thresholds_2d" title="confidence_thresholding_2thresholds_2d" class="md-nav__link">
    confidence_thresholding_2thresholds_2d
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#confidence_thresholding_2thresholds_3d" title="confidence_thresholding_2thresholds_3d" class="md-nav__link">
    confidence_thresholding_2thresholds_3d
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#binary-threshold-vs-metric" title="Binary Threshold vs. Metric" class="md-nav__link">
    Binary Threshold vs. Metric
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#binary_threshold_vs_metric" title="binary_threshold_vs_metric" class="md-nav__link">
    binary_threshold_vs_metric
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#roc-curves" title="ROC Curves" class="md-nav__link">
    ROC Curves
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#roc_curves" title="roc_curves" class="md-nav__link">
    roc_curves
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#roc_curves_from_test_stats" title="roc_curves_from_test_stats" class="md-nav__link">
    roc_curves_from_test_stats
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#calibration-plot" title="Calibration Plot" class="md-nav__link">
    Calibration Plot
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#calibration_1_vs_all" title="calibration_1_vs_all" class="md-nav__link">
    calibration_1_vs_all
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#calibration_multiclass" title="calibration_multiclass" class="md-nav__link">
    calibration_multiclass
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
          <li class="md-nav__item">
  <a href="#class-frequency-vs-f1-score" title="Class Frequency vs. F1 score" class="md-nav__link">
    Class Frequency vs. F1 score
  </a>
  
    <nav class="md-nav">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#frequency_vs_f1" title="frequency_vs_f1" class="md-nav__link">
    frequency_vs_f1
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
      
      
      
      
    </ul>
  
</nav>
                  </div>
                </div>
              </div>
            
          
          <div class="md-content">
            <article class="md-content__inner md-typeset">
              
                
                  <a href="https://github.com/uber/ludwig/edit/master/docs/user_guide.md" title="Edit this page" class="md-icon md-content__icon">&#xE3C9;</a>
                
                
                  <h1>User Guide</h1>
                
                <h2 id="command-line-interface">Command Line Interface<a class="headerlink" href="#command-line-interface" title="Permanent link">&para;</a></h2>
<p>Ludwig provides six command line interface entry points</p>
<ul>
<li>train</li>
<li>predict</li>
<li>experiment</li>
<li>visualize</li>
<li>collect_weights</li>
<li>collect_activations</li>
</ul>
<p>They are described in detail below.</p>
<h3 id="train">train<a class="headerlink" href="#train" title="Permanent link">&para;</a></h3>
<p>This command lets you train a model from your data.
You can call it with:</p>
<div class="codehilite"><pre><span></span>ludwig train [options]
</pre></div>


<p>or with</p>
<div class="codehilite"><pre><span></span>python -m ludwig.train [options]
</pre></div>


<p>from within Ludwig's main directory.</p>
<p>These are the available arguments:</p>
<div class="codehilite"><pre><span></span>usage: ludwig train [options]

This script trains a model.

optional arguments:
  -h, --help            show this help message and exit
  --output_directory OUTPUT_DIRECTORY
                        directory that contains the results
  --experiment_name EXPERIMENT_NAME
                        experiment name
  --model_name MODEL_NAME
                        name for the model
  --data_csv DATA_CSV   input data CSV file. If it has a split column, it will
                        be used for splitting (0: train, 1: validation, 2:
                        test), otherwise the dataset will be randomly split
  --data_train_csv DATA_TRAIN_CSV
                        input train data CSV file
  --data_validation_csv DATA_VALIDATION_CSV
                        input validation data CSV file
  --data_test_csv DATA_TEST_CSV
                        input test data CSV file
  --data_hdf5 DATA_HDF5
                        input data HDF5 file. It is an intermediate preprocess
                        version of the input CSV created the first time a CSV
                        file is used in the same directory with the same name
                        and a hdf5 extension
  --data_train_hdf5 DATA_TRAIN_HDF5
                        input train data HDF5 file. It is an intermediate
                        preprocess version of the input CSV created the first
                        time a CSV file is used in the same directory with the
                        same name and a hdf5 extension
  --data_validation_hdf5 DATA_VALIDATION_HDF5
                        input validation data HDF5 file. It is an intermediate
                        preprocess version of the input CSV created the first
                        time a CSV file is used in the same directory with the
                        same name and a hdf5 extension
  --data_test_hdf5 DATA_TEST_HDF5
                        input test data HDF5 file. It is an intermediate
                        preprocess version of the input CSV created the first
                        time a CSV file is used in the same directory with the
                        same name and a hdf5 extension
  --train_set_metadata_json TRAIN_SET_METADATA_JSON
                        input train set metadata JSON file. It is an intermediate
                        preprocess file containing the mappings of the input
                        CSV created the first time a CSV file is used in the
                        same directory with the same name and a json extension
  -sspi, --skip_save_processed_input
                        skips saving intermediate HDF5 and JSON files
  -md MODEL_DEFINITION, --model_definition MODEL_DEFINITION
                        model definition
  -mdf MODEL_DEFINITION_FILE, --model_definition_file MODEL_DEFINITION_FILE
                        YAML file describing the model. Ignores
                        --model_hyperparameters
  -mlp MODEL_LOAD_PATH, --model_load_path MODEL_LOAD_PATH
                        path of a pretrained model to load as initialization
  -mrp MODEL_RESUME_PATH, --model_resume_path MODEL_RESUME_PATH
                        path of a the model directory to resume training of
  -sspw SKIP_SAVE_PROGRESS_WEIGHTS, --skip_save_progress_weights SKIP_SAVE_PROGRESS_WEIGHTS
                        doesn&#39;t save weights after each epoch. By default
                        ludwig saves weights after each epoch for enabling
                        resuming of training, but if the model is really big
                        that can be time consuming and will save twice as much
                        space, use this parameter to skip it.
  -rs RANDOM_SEED, --random_seed RANDOM_SEED
                        a random seed that is going to be used anywhere there
                        is a call to a random number generator: data
                        splitting, parameter initialization and training set
                        shuffling
  -g GPUS [GPUS ...], --gpus GPUS [GPUS ...]
                        list of gpus to use
  -gf GPU_FRACTION, --gpu_fraction GPU_FRACTION
                        fraction of gpu memory to initialize the process with
  -dbg, --debug         enables debugging mode
  -l {critical,error,warning,info,debug,notset}, --logging_level {critical,error,warning,info,debug,notset}
                        the level of logging to use
</pre></div>


<p>When Ludwig trains a model it creates two intermediate files, one HDF5 and one JSON.
The HDF5 file contains the data mapped to numpy ndarrays, while the JSON file contains the mappings from the values in the tensors to their original labels.</p>
<p>For instance, for a categorical feature with 3 possible values, the HDF5 file will contain integers from 0 to 3 (with 0 being a <code>&lt;UNK&gt;</code> category), while the JSON file will contain a <code>idx2str</code> list containing all tokens (<code>[&lt;UNK&gt;, label_1, label_2, label_3]</code>), a <code>str2idx</code> dictionary (<code>{"&lt;UNK&gt;": 0, "label_1": 1, "label_2": 2, "label_3": 3}</code>) and a <code>str2freq</code> dictionary (<code>{"&lt;UNK&gt;": 0, "label_1": 93, "label_2": 55, "label_3": 24}</code>).</p>
<p>The reason to have those  intermediate files is two-fold: on one hand, if you are going to train your model again Ludwig will try to load them instead of recomputing all tensors, which saves a consistent amount of time, and on the other hand when you want to use your model to predict, data has to be mapped to tensors in exactly the same way it was mapped during training, so you'll be required to load the JSON metadata file in the <code>predict</code> command.
The way this works is: the first time you provide a CSV (<code>--data_csv</code>), the HDF5 and JSON files are created, from the second time on Ludwig will lod them instead of the CSV even if you specify the CSV (it looks in the same directory for files names in the same way but with a different extension), finally you can directly specify the HDF5 and JSON files (<code>--data_hdf5</code> and <code>--metadata_json</code>).</p>
<p>As the mapping from raw data to tensors depends on the type of feature that you specify in your model definition, if you change type (for instance from <code>sequential</code> to <code>text</code>) you also have to redo the preprocessing, which is achieved by deleting the HDF5 and JSON files.
Alternatively you can skip saving the HDF5 and JSON files specifying <code>--skip_save_processed_input</code>.</p>
<p>Splitting between train, validation and test set can be done in several ways.
This allows for a few possible input data scenarios:</p>
<ul>
<li>
<p>one single CSV file is provided (<code>-data_csv</code>). In this case if the csv contains a <code>split</code> column with values <code>0</code> for training, <code>1</code> for validation and <code>2</code> for test, this split will be used. If you want to ignore the split column and perform a random split, use a <code>force_split</code> argument in the model definition. In the case when there is no split column, a random <code>70-20-10</code> split will be performed. You can set the percentages and specify if you want stratified sampling in the model definition preprocessing section.</p>
</li>
<li>
<p>you can provide separate train, validation and test CSVs (<code>--data_train_csv</code>, <code>--data_validation_csv</code>, <code>--data_test_csv</code>).</p>
</li>
<li>
<p>the HDF5 and JSON file indications specified in the case of a single CSV file apply also in the multiple files case (<code>--data_train_hdf5</code>, <code>--data_validation_hdf5</code>, <code>--data_test_hdf5</code>), with the only difference that you need to specify only one JSON file (<code>--metadata_json</code>) instead of three.
The validation set is optional, but if absent the training wil continue until the end of the training epochs, while when there's a validation set the default behavior is to perform early stopping after the validation measure doesn't improve for a a certain amount of epochs.
The test set is optional too.</p>
</li>
</ul>
<p>Other optional arguments are <code>--output_directory</code>, <code>--experiment_name</code> and <code>--model name</code>.
By default the output directory is <code>./results</code>.
That directory will contain a directory named <code>[experiment_name]_[model_name]_0</code> if model name and experiment name are specified.
If the same combination of experiment and model name is used again, the integer at the end of the name wil be increased.
If neither of them is specified the directory will be named <code>run_0</code>.
The directory will will contain</p>
<ul>
<li><code>description.json</code> - a file containing a description of the training process with all the information to reproduce it.</li>
<li><code>training_statistics.json</code> which contains records of all measures and losses for each epoch.</li>
<li><code>model</code> - a directory containing model hyperparameters, weights, checkpoints and logs (for TensorBoard).</li>
</ul>
<p>The model definition can be provided either as a string (<code>--model_definition</code>) or as YAML file (<code>--model_definition_file</code>).
Details on how to write your model definition are provided in the <a href="#model-definition">Model Definition</a> section.</p>
<p>During training Ludwig saves two sets of weights for the model, one that is the weights at the end of the epoch where the best performance on the validation measure was achieved and one that is the weights at the end of the latest epoch.
The reason for keeping the second set is to be able to resume training in case the training process gets interrupted somehow.</p>
<p>To resume training using the latest weights and the whole history of progress so far you have to specify the <code>--model_resume_path</code> argument.
You can avoid saving the latest weights and the overall progress so far by using the argument <code>--skip_save_progress_weights</code>, but you will not be able to resume it afterwards.
Another available option is to load a previously trained model as an initialization for a new training process.
In this case Ludwig will start a new training process, without knowing any progress of the previous model, no training statistics, nor the number of epochs the model has been trained on so far.
It's not resuming training, just initializing training with a previously trained model with the same model definition, and it is accomplished through the <code>--model_resume_path</code> argument.</p>
<p>You can specify a random sed to be used by the python environment, python random package, numpy and TensorFlow with the <code>--random_seed</code> argument.
This is useful for reproducibility.
Be aware that due to asynchronicity in the TensorFlow GPU execution, when training on GPU results may not be reproducible.</p>
<p>You can manage which GPUs on your machine are used with the <code>--gpus</code> argument, which accepts a string identical to the format of <code>CUDA_VISIBLE_DEVICES</code> environment variable, namely a list of integers separated by comma.
You can also specify the fraction of the GPU memory that will be initially assigned to TensorFlow with <code>--gpu_fraction</code>.
By default it is 1.0, but you can set it, for instance, to 0.2 to use only 1/5 of the available memory.
If TensorFlow will need more GPU memory it will try to increase this amount.</p>
<p>Finally the <code>--logging_level</code> argument lets you set the amount of logging that you want to see during training and the <code>--debug</code> argument turns on TensorFlow's <code>tfdbg</code>. Be careful when doing so, as it will help in catching errors, in particular infs and NaNs but it will consume much more memory.</p>
<p>Example:</p>
<div class="codehilite"><pre><span></span>ludwig train --data_csv reuters-allcats.csv --model_definition &quot;{input_features: [{name: text, type: text, encoder: parallel_cnn, level: word}], output_features: [{name: class, type: category}]}&quot;
</pre></div>


<h3 id="predict">predict<a class="headerlink" href="#predict" title="Permanent link">&para;</a></h3>
<p>This command lets you use a previously trained model to predict on new data.
You can call it with:</p>
<div class="codehilite"><pre><span></span>ludwig predict [options]
</pre></div>


<p>or with</p>
<div class="codehilite"><pre><span></span>python -m ludwig.predict [options]
</pre></div>


<p>from within Ludwig's main directory.</p>
<p>These are the available arguments:</p>
<div class="codehilite"><pre><span></span>usage: ludwig predict [options]

This script loads a pretrained model and uses it to predict.

optional arguments:
  -h, --help            show this help message and exit
  --data_csv DATA_CSV   input data CSV file. If it has a split column, it will
                        be used for splitting (0: train, 1: validation, 2:
                        test), otherwise the dataset will be randomly split
  --data_hdf5 DATA_HDF5
                        input data HDF5 file. It is an intermediate preprocess
                        version of the input CSV created the first time a CSV
                        file is used in the same directory with the same name
                        and a hdf5 extension
  --train_set_metadata_json TRAIN_SET_METADATA_JSON
                        input train set metadata JSON file. It is an intermediate
                        preprocess file containing the mappings of the input
                        CSV created the first time a CSV file is used in the
                        same directory with the same name and a json extension
  -s {training,validation,test,full}, --split {training,validation,test,full}
                        the split to test the model on
  -m MODEL_PATH, --model_path MODEL_PATH
                        model to load
  -od OUTPUT_DIRECTORY, --output_directory OUTPUT_DIRECTORY
                        directory that contains the results
  -ssuo, --skip_save_unprocessed_output
                        skips saving intermediate NPY output files
  -bs BATCH_SIZE, --batch_size BATCH_SIZE
                        size of batches
  -op, --only_predictions
                        skip metrics calculation
  -g GPUS, --gpus GPUS  list of gpu to use
  -gf GPU_FRACTION, --gpu_fraction GPU_FRACTION
                        fraction of gpu memory to initialize the process with
  -dbg, --debug         enables debugging mode
  -l {critical,error,warning,info,debug,notset}, --logging_level {critical,error,warning,info,debug,notset}
                        the level of logging to use
</pre></div>


<p>The same distinction between CSV files and HDF5 / JSON files explained in the <a href="#train">train</a> section also applies here.
In either case, the JSON metadata file obtained during training is needed in order to map the new data into tensors.
If the new data contains a split column, you can specify which split to use to calculate the predictions with the <code>--split</code> argument. By default it's <code>full</code> which means all the splits will be used.</p>
<p>A model to load is needed, and yo can specify its path with the <code>--model_path</code> argument.
If you trained a model previously and got the results in, for instance, <code>./results/experiment_run_0</code>, you have to specify <code>./results/experiment_run_0/model</code> for using it to predict.</p>
<p>You can specify an output directory with the argument <code>--output-directory</code>, by default it will be <code>./result_0</code>, with increasing numbers if a directory with the same name is present.</p>
<p>The directory will contain a prediction CSV file and a probability CSV file for each output feature, together with raw NPY files containing raw tensors and a <code>predict_stats.json</code> file containing all prediction statistics.
By specifying the argument <code>--only_prediction</code> you will not get the statistics. This parameter is needed in the case your data doesn't contain ground truth output values, and thus computing the prediction statistics would be impossible.
If you receive an error regarding a missing output feature column in your data, you probably forgot to specify this argument.
You can moreover specify not to save the raw NPY output files with the argument <code>skip_save_unprocessed_output</code>.</p>
<p>A specific batch size for speeding up the prediction can be specified using the argument <code>--batch_size</code>.</p>
<p>Finally the <code>--logging_level</code>, <code>--debug</code> and <code>--gpus</code> related arguments behave exactly like described in the train command section.</p>
<p>Example:</p>
<div class="codehilite"><pre><span></span>ludwig predict --data_csv reuters-allcats.csv --model_path results/experiment_run_0/model/
</pre></div>


<h3 id="experiment">experiment<a class="headerlink" href="#experiment" title="Permanent link">&para;</a></h3>
<p>This command combines training and prediction into a single handy command.
You can call it with:</p>
<div class="codehilite"><pre><span></span>ludwig experiment [options]
</pre></div>


<p>or with</p>
<div class="codehilite"><pre><span></span>python -m ludwig.experiment [options]
</pre></div>


<p>from within Ludwig's main directory.</p>
<p>These are the available arguments:</p>
<div class="codehilite"><pre><span></span>usage: ludwig experiment [options]

This script trains and tests a model.

optional arguments:
  -h, --help            show this help message and exit
  --output_directory OUTPUT_DIRECTORY
                        directory that contains the results
  --experiment_name EXPERIMENT_NAME
                        experiment name
  --model_name MODEL_NAME
                        name for the model
  --data_csv DATA_CSV   input data CSV file. If it has a split column, it will
                        be used for splitting (0: train, 1: validation, 2:
                        test), otherwise the dataset will be randomly split
  --data_train_csv DATA_TRAIN_CSV
                        input train data CSV file
  --data_validation_csv DATA_VALIDATION_CSV
                        input validation data CSV file
  --data_test_csv DATA_TEST_CSV
                        input test data CSV file
  --data_hdf5 DATA_HDF5
                        input data HDF5 file. It is an intermediate preprocess
                        version of the input CSV created the first time a CSV
                        file is used in the same directory with the same name
                        and a hdf5 extension
  --data_train_hdf5 DATA_TRAIN_HDF5
                        input train data HDF5 file. It is an intermediate
                        preprocess version of the input CSV created the first
                        time a CSV file is used in the same directory with the
                        same name and a hdf5 extension
  --data_validation_hdf5 DATA_VALIDATION_HDF5
                        input validation data HDF5 file. It is an intermediate
                        preprocess version of the input CSV created the first
                        time a CSV file is used in the same directory with the
                        same name and a hdf5 extension
  --data_test_hdf5 DATA_TEST_HDF5
                        input test data HDF5 file. It is an intermediate
                        preprocess version of the input CSV created the first
                        time a CSV file is used in the same directory with the
                        same name and a hdf5 extension
  --train_set_metadata_json TRAIN_SET_METADATA_JSON
                        input train set metadata JSON file. It is an intermediate
                        preprocess file containing the mappings of the input
                        CSV created the first time a CSV file is used in the
                        same directory with the same name and a json extension
  -sspi, --skip_save_processed_input
                        skips saving intermediate HDF5 and JSON files
  -ssuo, --skip_save_unprocessed_output
                        skips saving intermediate NPY output files
  -md MODEL_DEFINITION, --model_definition MODEL_DEFINITION
                        model definition
  -mdf MODEL_DEFINITION_FILE, --model_definition_file MODEL_DEFINITION_FILE
                        YAML file describing the model. Ignores
                        --model_hyperparameters
  -mlp MODEL_LOAD_PATH, --model_load_path MODEL_LOAD_PATH
                        path of a pretrained model to load as initialization
  -mrp MODEL_RESUME_PATH, --model_resume_path MODEL_RESUME_PATH
                        path of a the model directory to resume training of
  -sspw SKIP_SAVE_PROGRESS_WEIGHTS, --skip_save_progress_weights SKIP_SAVE_PROGRESS_WEIGHTS
                        doesn&#39;t save weights after each epoch. By default
                        Ludwig saves weights after each epoch for enabling
                        resuming of training, but if the model is really big
                        that can be time consuming and will use twice as much
                        storage space, use this parameter to skip it.
  -rs RANDOM_SEED, --random_seed RANDOM_SEED
                        a random seed that is going to be used anywhere there
                        is a call to a random number generator: data
                        splitting, parameter initialization and training set
                        shuffling
  -g GPUS [GPUS ...], --gpus GPUS [GPUS ...]
                        list of gpus to use
  -gf GPU_FRACTION, --gpu_fraction GPU_FRACTION
                        fraction of gpu memory to initialize the process with
  -dbg, --debug         enables debugging mode
  -l {critical,error,warning,info,debug,notset}, --logging_level {critical,error,warning,info,debug,notset}
                        the level of logging to use
</pre></div>


<p>The parameters combine parameters from both <a href="#train">train</a> and <a href="#predict">predict</a> so please refer to those sections for an in depth explanation.
The output directory will contain the outputs both commands produce.</p>
<p>Example:</p>
<div class="codehilite"><pre><span></span>ludwig experiment --data_csv reuters-allcats.csv --model_definition &quot;{input_features: [{name: text, type: text, encoder: parallel_cnn, level: word}], output_features: [{name: class, type: category}]}&quot;
</pre></div>


<h3 id="visualize">visualize<a class="headerlink" href="#visualize" title="Permanent link">&para;</a></h3>
<p>This command lets you visualize training and prediction statistics, alongside with comparing different models performances and predictions.
You can call it with:</p>
<div class="codehilite"><pre><span></span>ludwig visualize [options]
</pre></div>


<p>or with</p>
<div class="codehilite"><pre><span></span>python -m ludwig.visualize [options]
</pre></div>


<p>from within Ludwig's main directory.</p>
<p>These are the available arguments:</p>
<div class="codehilite"><pre><span></span>usage: ludwig visualize [options]

This script analyzes results and shows some nice plots.

optional arguments:
  -h, --help            show this help message and exit
  -r RAW_DATA, --raw_data RAW_DATA
                        raw data file
  -g GROUND_TRUTH, --ground_truth GROUND_TRUTH
                        ground truth file
  -gm GROUND_TRUTH_METADATA, --ground_truth_metadata GROUND_TRUTH_METADATA
                        input metadata JSON file
  -v {compare_performance,compare_classifiers_performance_from_prob,compare_classifiers_performance_from_pred,compare_classifiers_performance_changing_k,compare_classifiers_performance_subset,compare_classifiers_predictions,compare_classifiers_predictions_distribution,confidence_thresholding,confidence_thresholding_2thresholds_3d,confidence_thresholding_data_vs_acc,confidence_thresholding_2thresholds_2d,confidence_thresholding_data_vs_acc_subset,confidence_thresholding_data_vs_acc_subset_per_class,binary_threshold_vs_metric,roc_curves,roc_curves_from_test_stats,data_vs_acc_subset,data_vs_acc_subset_per_class,calibration_1_vs_all,calibration_multiclass,confusion_matrix,compare_classifiers_multiclass_multimetric,frequency_vs_f1,learning_curves}, --visualization {compare_performance,compare_classifiers_performance_from_prob,compare_classifiers_performance_from_pred,compare_classifiers_performance_changing_k,compare_classifiers_performance_subset,compare_classifiers_predictions,compare_classifiers_predictions_distribution,confidence_thresholding,confidence_thresholding_2thresholds_3d,confidence_thresholding_data_vs_acc,confidence_thresholding_2thresholds_2d,confidence_thresholding_data_vs_acc_subset,confidence_thresholding_data_vs_acc_subset_per_class,binary_threshold_vs_metric,roc_curves,roc_curves_from_test_stats,data_vs_acc_subset,data_vs_acc_subset_per_class,calibration_1_vs_all,calibration_multiclass,confusion_matrix,compare_classifiers_multiclass_multimetric,frequency_vs_f1,learning_curves}
                        type of visualization
  -f FIELD, --field FIELD
                        field containing ground truth
  -tf THRESHOLD_FIELDS [THRESHOLD_FIELDS ...], --threshold_fields THRESHOLD_FIELDS [THRESHOLD_FIELDS ...]
                        fields for 2d threshold
  -pred PREDICTIONS [PREDICTIONS ...], --predictions PREDICTIONS [PREDICTIONS ...]
                        predictions files
  -prob PROBABILITIES [PROBABILITIES ...], --probabilities PROBABILITIES [PROBABILITIES ...]
                        probabilities files
  -tes TRAINING_STATS [TRAINING_STATS ...], --training_stats TRAINING_STATS [TRAINING_STATS ...]
                        training stats files
  -trs TEST_STATS [TEST_STATS ...], --test_stats TEST_STATS [TEST_STATS ...]
                        test stats files
  -alg ALGORITHMS [ALGORITHMS ...], --algorithms ALGORITHMS [ALGORITHMS ...]
                        names of the algorithms (for better graphs)
  -tn TOP_N_CLASSES [TOP_N_CLASSES ...], --top_n_classes TOP_N_CLASSES [TOP_N_CLASSES ...]
                        number of classes to plot
  -k TOP_K, --top_k TOP_K
                        number of elements in the ranklist to consider
  -ll LABELS_LIMIT, --labels_limit LABELS_LIMIT
                        maximum numbers of labels. If labels in dataset are
                        higher than this number, &quot;rare&quot; label
  -ss {ground_truth,predictions}, --subset {ground_truth,predictions}
                        type of subset filtering
  -n, --normalize       normalize rows in confusion matrix
  -m METRICS [METRICS ...], --metrics METRICS [METRICS ...]
                        metrics to dispay in threshold_vs_metric
  -pl POSITIVE_LABEL, --positive_label POSITIVE_LABEL
                        label of the positive class for the roc curve
  -l {critical,error,warning,info,debug,notset}, --logging_level {critical,error,warning,info,debug,notset}
                        the level of logging to use
</pre></div>


<p>As the <code>--visualization</code> parameters suggests, there is a vast number of visualizations readily available.
Each of them requires a different subset of this command's arguments, so they will be described one by one in the <a href="#visualizations">Visualizations</a> section.</p>
<h3 id="collect_weights">collect_weights<a class="headerlink" href="#collect_weights" title="Permanent link">&para;</a></h3>
<p>This command lets you load a pre-trained model and collect the tensors with a specific name in order to save them in a NPY format.
This may be useful in order to visualize the learned weights (for instance collecting embedding matrices) and for some post-hoc analyses.
You can call it with:</p>
<div class="codehilite"><pre><span></span>ludwig collect_weights [options]
</pre></div>


<p>or with</p>
<div class="codehilite"><pre><span></span>python -m ludwig.collect weights [options]
</pre></div>


<p>from within Ludwig's main directory.</p>
<p>These are the available arguments:</p>
<div class="codehilite"><pre><span></span>usage: ludwig collect_weights [options]

This script loads a pretrained model and uses it collect weights.

optional arguments:
  -h, --help            show this help message and exit
  -m MODEL_PATH, --model_path MODEL_PATH
                        model to load
  -t TENSORS [TENSORS ...], --tensors TENSORS [TENSORS ...]
                        tensors to collect
  -od OUTPUT_DIRECTORY, --output_directory OUTPUT_DIRECTORY
                        directory that contains the results
  -dbg, --debug         enables debugging mode
  -l {critical,error,warning,info,debug,notset}, --logging_level {critical,error,warning,info,debug,notset}
                        the level of logging to use
</pre></div>


<p>The three most important arguments are <code>--model_path</code> where you have to specify the path of the model to load, <code>--tensors</code> that lets you specify a list of tensor names in the TensorFlow graph that contain the weights you want to collect, and finally <code>--output_directory</code> that lets you specify where the NPY files (one for each tensor name specified) will be saved.</p>
<p>In order to figure out the names fo the tensors containing the weights you want to collect, the best way is to inspect the graph of the model with TensorBoard.</p>
<div class="codehilite"><pre><span></span>tensorboard --logdir /path/to/model/log
</pre></div>


<h3 id="collect_activations">collect_activations<a class="headerlink" href="#collect_activations" title="Permanent link">&para;</a></h3>
<p>This command lets you load a pre-trained model and input data and collects the values of activations contained in tensors with a specific name in order to save them in a NPY format.
This may be useful in order to visualize the activations (for instance collecting last layer's activations as embeddings representations of the input datapoint) and for some post-hoc analyses.
You can call it with:</p>
<div class="codehilite"><pre><span></span>ludwig collect_activations [options]
</pre></div>


<p>or with</p>
<div class="codehilite"><pre><span></span>python -m ludwig.collect activations [options]
</pre></div>


<p>from within Ludwig's main directory.</p>
<p>These are the available arguments:</p>
<div class="codehilite"><pre><span></span>usage: ludwig collect_activations [options]

This script loads a pretrained model and uses it collect tensors for each
datapoint in the dataset.

optional arguments:
  -h, --help            show this help message and exit
  --data_csv DATA_CSV   input data CSV file
  --data_hdf5 DATA_HDF5
                        input data HDF5 file
  -s {training,validation,test,full}, --split {training,validation,test,full}
                        the split to test the model on
  -m MODEL_PATH, --model_path MODEL_PATH
                        model to load
  -t TENSORS [TENSORS ..], --tensors TENSORS [TENSORS ..]
                        tensors to collect
  -od OUTPUT_DIRECTORY, --output_directory OUTPUT_DIRECTORY
                        directory that contains the results
  -bs BATCH_SIZE, --batch_size BATCH_SIZE
                        size of batches
  -g GPUS, --gpus GPUS  list of gpu to use
  -gf GPU_FRACTION, --gpu_fraction GPU_FRACTION
                        fraction of gpu memory to initialize the process with
  -dbg, --debug         enables debugging mode
  -l {critical,error,warning,info,debug,notset}, --logging_level {critical,error,warning,info,debug,notset}
                        the level of logging to use
</pre></div>


<p>The data related and runtime related arguments (GPUs, batch size, etc.) are the same used in <a href="#predict">predict</a>, you can refer to that section for an explanation.
The collect specific arguments  <code>--model_path</code>, <code>--tensors</code>  and <code>--output_directory</code> are the same used in <a href="#collect_weights">collect_weights</a>, you can refer to that section for an explanation.</p>
<p>In order to figure out the names fo the tensors containing the activations you want to collect, the best way is to inspect the graph of the model with TensorBoard.</p>
<div class="codehilite"><pre><span></span>tensorboard --logdir /path/to/model/log
</pre></div>


<h2 id="data-preprocessing">Data Preprocessing<a class="headerlink" href="#data-preprocessing" title="Permanent link">&para;</a></h2>
<p>Ludwig data preprocessing maps raw data coming in CSV format into an HDF5 file containing tensors and a JSON file containing mappings from strings to tensors when needed.
This mapping is performed when a CSV is provided as input and both HDF5 and JSON files are saved in the same directory as the input CSV, unless the argument <code>--skip_save_processed_input</code> is used (both in <code>train</code> and <code>experiment</code> commands).
The reason to save those files is both to provide a cache and avoid performing the preprocessing again (as, depepnding on the type of features involved, it could be time consuming) and to provide the needed mappings to be able to map unseen data into tensors.</p>
<p>The preprocessing process is personalizable to fit the specifics of your data format, but the basic assumption is always that your CSV files contains one row for each datapoint and one column for each feature (either input or output), and that you are able to determine the type of that column among the ones supported by Ludwig.
The reason for that is that each data type is mapped into tensors in a different way and expects the content to be formatted in a specific way.
Different datatypes may have different formatters that format the values of a cell.</p>
<p>For instance the value of a cell of a sequence feature column by default is managed by a <code>space</code> formatter, that splits the content of the value into a list of strings using space.</p>
<table>
<thead>
<tr>
<th>before formatter</th>
<th>after formatter</th>
</tr>
</thead>
<tbody>
<tr>
<td>"token3 token4 token2"</td>
<td>[token3, token4, token2]</td>
</tr>
<tr>
<td>"token3 token1"</td>
<td>[token3, token1]</td>
</tr>
</tbody>
</table>
<p>Then a list <code>idx2str</code> and two dictionaries <code>str2idx</code> and <code>str2freq</code> are created containing all the tokens in all the lists obtained by splitting all the rows of the column and an integer id is assigned to each of them (in order of frequency).</p>
<div class="codehilite"><pre><span></span><span class="p">{</span>
    <span class="nt">&quot;column_name&quot;</span><span class="p">:</span> <span class="p">{</span>
        <span class="nt">&quot;idx2str&quot;</span><span class="p">:</span> <span class="p">[</span>
            <span class="s2">&quot;&lt;PAD&gt;&quot;</span><span class="p">,</span>
            <span class="s2">&quot;&lt;UNK&gt;&quot;</span><span class="p">,</span>
            <span class="s2">&quot;token3&quot;</span><span class="p">,</span>
            <span class="s2">&quot;token2&quot;</span><span class="p">,</span>
            <span class="s2">&quot;token4&quot;</span><span class="p">,</span>
            <span class="s2">&quot;token1&quot;</span>
        <span class="p">],</span>
        <span class="nt">&quot;str2idx&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;&lt;PAD&gt;&quot;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
            <span class="nt">&quot;&lt;UNK&gt;&quot;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
            <span class="nt">&quot;token3&quot;</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span>
            <span class="nt">&quot;token2&quot;</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span>
            <span class="nt">&quot;token4&quot;</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span>
            <span class="nt">&quot;token1&quot;</span><span class="p">:</span> <span class="mi">5</span>
        <span class="p">},</span>
        <span class="nt">&quot;str2freq&quot;</span><span class="p">:</span> <span class="p">{</span>
            <span class="nt">&quot;&lt;PAD&gt;&quot;</span><span class="p">:</span>  <span class="mi">0</span><span class="p">,</span>
            <span class="nt">&quot;&lt;UNK&gt;&quot;</span><span class="p">:</span>  <span class="mi">0</span><span class="p">,</span>
            <span class="nt">&quot;token3&quot;</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span>
            <span class="nt">&quot;token2&quot;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
            <span class="nt">&quot;token4&quot;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
            <span class="nt">&quot;token1&quot;</span><span class="p">:</span> <span class="mi">1</span>
        <span class="p">}</span>
    <span class="p">}</span>
<span class="p">}</span>
</pre></div>


<p>Finally a numpy matrix is created with sizes <code>n x l</code> where <code>n</code> is the number of rows in the column and <code>l</code> is the minimum of the longest tokenized list and a <code>max_lenght</code> parameter that can be set.
All sequences shorter than <code>l</code> are padded on the right (but this behavior may also be modified through a parameter).</p>
<table>
<thead>
<tr>
<th>after formatter</th>
<th>numpy matrix</th>
</tr>
</thead>
<tbody>
<tr>
<td>[token3, token4, token2]</td>
<td>2 4 3</td>
</tr>
<tr>
<td>[token3, token1]</td>
<td>2 5 0</td>
</tr>
</tbody>
</table>
<p>The final result matrix is saved in the HDF5 the name of the original column in the CSV as key, while the mapping from token to integer ID (and its inverse mapping) is saved in the JSON file.</p>
<p>Each datatype is preprocessed in a different way, using different parameters and different formatters.
Details on how to set those parameters for each feature type and for each specific feature will be described in the <a href="#preprocessing">Model Definition - Preprocessing</a> section.</p>
<p><code>Binary</code> features are directly transformed into a binary valued vector of length <code>n</code> (where <code>n</code> is the size of the dataset) and added to HDF5 with a key that reflects the name of column in the CSV.
No additional information about them is available in the JSON metadata file.</p>
<p><code>Numerical</code> features are directly transformed into a float valued vector of length <code>n</code> (where <code>n</code> is the size of the dataset) and added to HDF5 with a key that reflects the name of column in the CSV.
No additional information about them is available in the JSON metadata file.</p>
<p><code>Category</code> features are transformed into into an integer valued vector of size <code>n</code> (where <code>n</code> is the size of the dataset) and added to HDF5 with a key that reflects the name of column in the CSV.
The way categories are mapped into integers consists in first collecting a dictionary of all the different category strings present in the column of the CSV, then rank them by frequency and then assign them an increasing integer ID from the most frequent to the most rare (with 0 being assigned to a <code>&lt;UNK&gt;</code> token).
The column name is added to the JSON file, with an associated dictionary containing
1. the mapping from integer to string (<code>idx2str</code>)
2. the mapping from string to id (<code>str2idx</code>)
3. the mapping from string to frequency (<code>str2freq</code>)
4. the size of the set of all tokens (<code>vocab_size</code>)
4. additional preprocessing information (by default how to fill missing values and what token to use to fill missing values)</p>
<p><code>Set</code> features are transformed into into a binary (int8 actually) valued matrix of size <code>n x l</code> (where <code>n</code> is the size of the dataset and <code>l</code> is the minimum of the size of the biggest set and a <code>max_size</code> parameter) and added to HDF5 with a key that reflects the name of column in the CSV.
The way sets are mapped into integers consists in first using a formatter to map from strings to sequences of set items (by default this is done by splitting on spaces).
Then a a dictionary of all the different set item strings present in the column of the CSV is collected, then they are ranked by frequency and an increasing integer ID is assigned to them from the most frequent to the most rare (with 0 being assigned to <code>&lt;PAD&gt;</code> used for padding and 1 assigned to <code>&lt;UNK&gt;</code> item).
The column name is added to the JSON file, with an associated dictionary containing
1. the mapping from integer to string (<code>idx2str</code>)
2. the mapping from string to id (<code>str2idx</code>)
3. the mapping from string to frequency (<code>str2freq</code>)
4. the maximum size of all sets (<code>max_set_size</code>)
5. additional preprocessing information (by default how to fill missing values and what token to use to fill missing values)</p>
<p><code>Bag</code> features are treated in the same way of set features, with the only difference being that the matrix had float values (frequencies).</p>
<p><code>Sequence</code> features are transformed into into an integer valued matrix of size <code>n x l</code> (where <code>n</code> is the size of the dataset and <code>l</code> is the minimum of the length of the longest sequence and a <code>sequence_length_limit</code> parameter) and added to HDF5 with a key that reflects the name of column in the CSV.
The way sets are mapped into integers consists in first using a formatter to map from strings to sequences of tokens (by default this is done by splitting on spaces).
Then a a dictionary of all the different token strings present in the column of the CSV is collected, then they are ranked by frequency and an increasing integer ID is assigned to them from the most frequent to the most rare (with 0 being assigned to <code>&lt;PAD&gt;</code> used for padding and 1 assigned to <code>&lt;UNK&gt;</code> item).
The column name is added to the JSON file, with an associated dictionary containing
1. the mapping from integer to string (<code>idx2str</code>)
2. the mapping from string to id (<code>str2idx</code>)
3. the mapping from string to frequency (<code>str2freq</code>)
4. the maximum length of all sequences (<code>sequence_length_limit</code>)
5. additional preprocessing information (by default how to fill missing values and what token to use to fill missing values)</p>
<p><code>Text</code> features are treated in the same way of sequence features, with a couple differences.
Two different formatting/splitting happen, one that splits at every character and one that uses a spaCy based tokenizer (and removes stopwords) are used, and two different key are added to the HDF5 file, one containing the matrix of characters and one containing the matrix of words.
The same thing happens in the JSON file, where there are dictionaries for mapping characters to integers (and the inverse) and words to integers (and their inverse).
In the model definition you are able to specify which level of representation to use, if the character level or the word level.</p>
<p><code>Timeseries</code> features are treated in the same way of sequence features, with the only difference being that the matrix in the HDF5 file does not have integer values, but float values.
Moreover, there is no need for any mapping in the JSON file.</p>
<p><code>Image</code> features are transformed into into a int8 valued tensor of size <code>n x h x w x c</code> (where <code>n</code> is the size of the dataset and <code>h x w</code> is a specific resizing of the image that can be set, and <code>c</code> is the number of color channels) and added to HDF5 with a key that reflects the name of column in the CSV.
The column name is added to the JSON file, with an associated dictionary containing preprocessing information about the sizes of the resizing.</p>
<h3 id="csv-format">CSV Format<a class="headerlink" href="#csv-format" title="Permanent link">&para;</a></h3>
<p>Ludwig uses Pandas under the hood to read the CSV files. Pandas tries to automatically identify the separator (generally <code>','</code>) from the data.
We are using <code>'\'</code> as the default escape character. For example, if <code>','</code> is the column separator and one of your data columns has a <code>','</code> in it, Pandas would fail to load the data properly.
To handle such cases, we expect your data columns to be escaped with backslashes (replace <code>','</code> in the data with <code>'\\,'</code>)</p>
<h2 id="data-postprocessing">Data Postprocessing<a class="headerlink" href="#data-postprocessing" title="Permanent link">&para;</a></h2>
<p>The JSON file obtained from preprocessing is used also for postprocessing: Ludwig models return output predictions and, depending on their datatype they are mapped back into the original space.
Numerical and timeseries are returned as they are, while category, set, sequence, and text features output integers, those integers are mapped back into the original tokens / names using the <code>idx2str</code> in the JSON file.
When you run <code>experiment</code> or <code>predict</code> you will find both a CSV file for each output containing the mapped predictions, a probability CSV file containing the probability of that prediction, a probabilities CSV file containing the probabilities for all alternatives (for instance, the probabilities of all the categories in case of a categorical feature).
You will also find the unmapped NPY files.
If you don't need them you can use the <code>--skip_save_unprocessed_output</code> argument.</p>
<h2 id="model-definition">Model Definition<a class="headerlink" href="#model-definition" title="Permanent link">&para;</a></h2>
<p>The model definition is the core of Ludwig.
It is a dictionary that contains all the information needed to build and train a Ludwig model.
I mixes ease of use, by means of reasonable defaults, with flexibility, by means of detailed control over the parameters of your model.
It is provided to both <code>experiment</code> and <code>train</code> commands either as a string (<code>--model_definition</code>) or as a file (<code>--model_definition_file</code>).
The string or the content of the file will be parsed by PyYAML into a dictionary in memory, so any style of YAML accepted by the parser is considered to be valid, so both multiline and oneline formats are accepted.
For instance a list of dictionaries can be written both as</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">mylist</span><span
                        class="p p-Indicator">:</span> <span class="p p-Indicator">[{</span><span class="nv">name</span><span
                        class="p p-Indicator">:</span> <span class="nv">item1</span><span class="p p-Indicator">,</span> <span
                        class="nv">score</span><span class="p p-Indicator">:</span> <span class="nv">2</span><span
                        class="p p-Indicator">},</span> <span class="p p-Indicator">{</span><span class="nv">name</span><span
                        class="p p-Indicator">:</span> <span class="nv">item2</span><span class="p p-Indicator">,</span> <span
                        class="nv">score</span><span class="p p-Indicator">:</span> <span class="nv">1</span><span
                        class="p p-Indicator">},</span> <span class="p p-Indicator">{</span><span class="nv">name</span><span
                        class="p p-Indicator">:</span> <span class="nv">item3</span><span class="p p-Indicator">,</span> <span
                        class="nv">score</span><span class="p p-Indicator">:</span> <span class="nv">4</span><span
                        class="p p-Indicator">}]</span>
</pre></div>


<p>or as:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">mylist</span><span
                        class="p p-Indicator">:</span>
    <span class="p p-Indicator">-</span>
        <span class="l l-Scalar l-Scalar-Plain">name</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">item1</span>
        <span class="l l-Scalar l-Scalar-Plain">score</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">2</span>
    <span class="p p-Indicator">-</span>
        <span class="l l-Scalar l-Scalar-Plain">name</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">item2</span>
        <span class="l l-Scalar l-Scalar-Plain">score</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">1</span>
    <span class="p p-Indicator">-</span>
        <span class="l l-Scalar l-Scalar-Plain">name</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">item3</span>
        <span class="l l-Scalar l-Scalar-Plain">score</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">4</span>
</pre></div>


<p>The structure of the model definition file is a dictionary with five keys:</p>
                <div class="codehilite"><pre><span></span><span
                        class="l l-Scalar l-Scalar-Plain">input_features</span><span
                        class="p p-Indicator">:</span> <span class="p p-Indicator">[]</span>
<span class="l l-Scalar l-Scalar-Plain">combiner</span><span class="p p-Indicator">:</span> <span class="p p-Indicator">{}</span>
<span class="l l-Scalar l-Scalar-Plain">output_features</span><span class="p p-Indicator">:</span> <span
                            class="p p-Indicator">[]</span>
<span class="l l-Scalar l-Scalar-Plain">training</span><span class="p p-Indicator">:</span> <span class="p p-Indicator">{}</span>
<span class="l l-Scalar l-Scalar-Plain">preprocessing</span><span class="p p-Indicator">:</span> <span
                            class="p p-Indicator">{}</span>
</pre></div>


<p>Only <code>input_features</code> and <code>output_features</code> are required, the other three fields have default values, but you are free to modify them.</p>
<h3 id="input-features">Input features<a class="headerlink" href="#input-features" title="Permanent link">&para;</a></h3>
<p>The <code>input_features</code> list contains a list of dictionaries, each of them containing two required fields <code>name</code> and <code>type</code>.
<code>name</code> is the name of the feature and is the same name of the column of the CSV input file, <code>type</code> is one of the supported datatypes.
Input features may have different ways to be encoded and the parameter to decide it is <code>encoder</code>.</p>
<p>All the other parameters you specify in an input feature will be passed as parameters to the function that build the encoder, and each encoder can have different parameters.</p>
<p>For instance a <code>sequence</code> feature can be encoded by a <code>stacked_cnn</code> or by and <code>rnn</code>, but only the <code>stacked_cnn</code> will accept the parameter <code>num_filters</code> while only the <code>rnn</code> will accept the parameter <code>bidirectional</code>.</p>
<p>A list of all the encoders available for all the datatypes alongside with the description of all parameters will be provided in the datatype-specific sections.
Some datatypes have only one type of encoder, so you are not required to specify it.</p>
<p>The role of the encoders is to map inputs into tensors, usually vectors in the case fo datatype without a temporal / sequential aspect, matrices in case there is a temporal / sequential aspect or higher rank tensors in case there is a spatial or a spatio-temporal aspect to the input data.</p>
<p>Different configurations of the same encoder may return a tensor with different rank, for instance a sequential encoder may return a vector of size <code>h</code> that is either the final vector of a sequence or the result of pooling over the sequence length, or it can return a matrix of size <code>l x h</code> where <code>l</code> is the length of the sequence and <code>h</code> is the hidden dimension if you specify the pooling reduce operation (<code>reduce_output</code>) to be <code>null</code>.
For the sake of simplicity you can imagine the output to be a vector in most of the cases, but there is a <code>reduce_output</code> parameter one can specify to change the default behavior.</p>
<p>An additional feature that ludwig provides is the option to have tied weights between different encoders.
For instance if my model takes two sentences as input and return the probability of their entailment, I may want to encode both sentences with the same encoder.
The way to do it is by specifying the <code>tied-weights</code> parameter of the second feature you define to be the name of the first feature you defined.</p>
                <div class="codehilite"><pre><span></span><span
                        class="l l-Scalar l-Scalar-Plain">input_features</span><span class="p p-Indicator">:</span>
    <span class="p p-Indicator">-</span>
        <span class="l l-Scalar l-Scalar-Plain">name</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sentence1</span>
        <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">text</span>
    <span class="p p-Indicator">-</span>
        <span class="l l-Scalar l-Scalar-Plain">name</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sentence2</span>
        <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">text</span>
        <span class="l l-Scalar l-Scalar-Plain">tied_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sentence1</span>
</pre></div>


<p>If you specify a name of an input feature that has not been defined yet, it will result in an error.
Also, in order to be able to have tied weights, all encoder parameters have to be identical between the two input features.</p>
<h3 id="combiner">Combiner<a class="headerlink" href="#combiner" title="Permanent link">&para;</a></h3>
<p>Combiners are part of the model that take all the outputs of the different input features and combine them in a single representation that is passed to the outputs.
You can specify which one to use in the <code>combiner</code> section of the model definition.
Different combiners implement different combination logic, but the default one <code>concat</code> just concatenates all outputs of input feature encoders and optionally passes the concatenation through fully connected layers, with the output of the last layer being forwarded to the outputs decoders.</p>
<div class="codehilite"><pre><span></span>+-----------+
|Input      |
|Feature 1  +-+
+-----------+ |            +---------+
+-----------+ | +------+   |Fully    |
|...        +---&gt;Concat+---&gt;Connected+-&gt;
+-----------+ | +------+   |Layers   |
+-----------+ |            +---------+
|Input      +-+
|Feature N  |
+-----------+
</pre></div>


<p>For the sake of simplicity you can imagine the both inputs and outputs are vectors in most of the cases, but there are <code>reduce_input</code> and <code>reduce_output</code> parameters to specify to change the default behavior.</p>
<h3 id="output-features">Output Features<a class="headerlink" href="#output-features" title="Permanent link">&para;</a></h3>
<p>The <code>output_features</code> list has the same structure of the <code>input_features</code> list: it is a list of dictionaries containing a <code>name</code> and a <code>type</code>.
They represent outputs / targets that you want your model to predict.
In most machine learning tasks you want to predict only one target variable, but in Ludwig you are allowed to specify as many outputs as you want and they are going to be optimized in a multi-task fashion, using a weighted sum of their losses as a combined loss to optimize.</p>
<p>Instead of having <code>encoders</code>, output features have <code>decoders</code>, but most of them have only one decoder so you don't have to specify it.</p>
<p>Decoders take the output of the combiner as input, process it further, for instance passing it through fully connected layers, and finally predict values and compute a loss and some measures (depending on the datatype different losses and measures apply).</p>
<p>Decoders have additional parameters, in particular <code>loss</code> that allows you to specify a different loss to optimize for this specific decoder, for instance numerical features support both <code>mean_squared_error</code> and <code>mean_absolute_error</code> as losses.
Details about the available decoders and losses alongside with the description of all parameters will be provided in the datatype-specific sections.</p>
<p>For the sake of simplicity you can imagine the input coming from the combiner to be a vector in most of the cases, but there is a <code>reduce_input</code> parameter one can specify to change the default behavior.</p>
<h3 id="output-features-dependencies">Output Features Dependencies<a class="headerlink" href="#output-features-dependencies" title="Permanent link">&para;</a></h3>
<p>An additional feature that Ludwig provides is the concept of dependency between <code>output_features</code>.
You can specify a list of output features as dependencies when you write the dictionary of a specific feature.
At model building time Ludwig checks that no cyclic dependency exists.
If you do so Ludwig will concatenate all the final representations before the prediction of those output features to the original input of the decoder.
The reason is that if different output features have a causal dependency, knowing which prediction has been made for one can help making the prediction of the other.</p>
<p>For instance if two output features are one coarse grained category and one fine-grained category that are in a hierarchical structure with each other, knowing the prediction made for coarse grained restricts the possible categories to predict for the fine-grained.
In this case the following model definition structure can be used:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">output_features</span><span
                        class="p p-Indicator">:</span>
    <span class="p p-Indicator">-</span>
        <span class="l l-Scalar l-Scalar-Plain">name</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">coarse_class</span>
        <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">category</span>
        <span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">2</span>
        <span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">64</span>
    <span class="p p-Indicator">-</span>
        <span class="l l-Scalar l-Scalar-Plain">name</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">fine_class</span>
        <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">category</span>
        <span class="l l-Scalar l-Scalar-Plain">dependencies</span><span class="p p-Indicator">:</span>
            <span class="p p-Indicator">-</span> <span class="l l-Scalar l-Scalar-Plain">coarse_class</span>
        <span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">1</span>
        <span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">64</span>
</pre></div>


<p>Assuming the input coming from the combiner has hidden dimension <code>h</code> 128, there are two fully connected layers that return a vector with hidden size 64 at the end of the <code>coarse_class</code> decoder (that vector will be used for the final layer before projecting in the output <code>coarse_class</code> space)
In the decoder of <code>fine_class</code>, the 64 dimensional vector of <code>coarse_class</code> will be concatenated to the combiner output vector, making a vector of hidden size 192 that will be passed through a fully connected layer and the 64 dimensional output will be used for the final layer before projecting in the output class space of the <code>fine_class</code>.</p>
<h3 id="training">Training<a class="headerlink" href="#training" title="Permanent link">&para;</a></h3>
<p>The <code>training</code> section of the model definition lets you specify some parameters of the training process, like for instance the number of epochs or the learning rate.</p>
<p>These are the available training parameters:</p>
<ul>
<li><code>batch_size</code> (default <code>128</code>): size of the batch used for training the model.</li>
<li><code>epochs</code> (default <code>200</code>): number of epochs the training process will run for.</li>
<li><code>early_stop</code> (default <code>10</code>): if there's a validation set, number of epochs of patience without an improvement on the validation measure before the training is stopped.</li>
<li><code>optimizer</code> (default <code>{type: adam, beta1: 0.9, beta2: 0.999, epsilon: 1e-08}</code>): which optimizer to use with the relative parameters. The available optimizers are: <code>sgd</code> (or <code>stochastic_gradient_descent</code>, <code>gd</code>, <code>gradient_descent</code>, they are all the same), <code>adam</code>, <code>adadelta</code>, <code>adagrad</code>, <code>adagradda</code>, <code>momentum</code>, <code>ftrl</code>, <code>proximalgd</code>, <code>proximaladagrad</code>, <code>rmsprop</code>. To know their parameters check <a href="https://www.tensorflow.org/api_docs/python/tf/keras/optimizers">TensorFlow's optimizer documentation</a>.</li>
<li><code>learning_rate</code> (default <code>0.001</code>): the learning rate to use.</li>
<li><code>decay</code> (default <code>false</code>): if to use exponential decay of the learning rate or not.</li>
<li><code>decay_rate</code> (default <code>0.96</code>): the rate of the exponential learning rate decay.</li>
<li><code>decay_steps</code> (default <code>10000</code>): the number of steps of the exponential learning rate decay.</li>
<li><code>staircase</code> (default <code>false</code>): decays the learning rate at discrete intervals.</li>
<li><code>regularization_lambda</code> (default <code>0</code>): the lambda parameter used for adding a l2 regularization loss to the overall loss.</li>
<li><code>dropout_rate</code> (default <code>0.0</code>): the probability to drop neurons in dropout. The <code>dropout_rate</code> is used throughout the whole model, but to decide which parts of the model will use it, use the <code>dropout</code> boolean parameter available in each encoder, combiner and decoder.</li>
<li><code>reduce_learning_rate_on_plateau</code> (default <code>0</code>): if there's a validation set, how many times to reduce the learning rate when a plateau of validation measure is reached.</li>
<li><code>reduce_learning_rate_on_plateau_patience</code> (default <code>5</code>): if there's a validation set, number of epochs of patience without an improvement on the validation measure before reducing the learning rate.</li>
<li><code>reduce_learning_rate_on_plateau_rate</code> (default <code>0.5</code>): if there's a validation set, the reduction rate of the learning rate.</li>
<li><code>increase_batch_size_on_plateau</code> (default <code>0</code>): if there's a validation set, how many times to increase the batch size when a plateau of validation measure is reached.</li>
<li><code>increase_batch_size_on_plateau_patience</code> (default <code>5</code>): if there's a validation set, number of epochs of patience without an improvement on the validation measure before increasing the learning rate.</li>
<li><code>increase_batch_size_on_plateau_rate</code> (default <code>2</code>): if there's a validation set, the increase rate of the batch size.</li>
<li><code>increase_batch_size_on_plateau_max</code> (default <code>512</code>):  if there's a validation set, the maximum value of batch size.</li>
<li><code>validation_field</code> (default <code>combined</code>): when there is more than one output feature, which one to use for computing if there was an improvement on validation. The measure to use to determine if there was an improvement can be set with the <code>validation_measure</code> parameter. Different datatypes have different available measures, refer to the datatype-specific section for more details. <code>combined</code> indicates the use the combination of all features. For instance the combination of <code>combined</code> and <code>loss</code> as measure uses a decrease in the combined loss of all output features to check for improvement on validation, while <code>combined</code> and <code>accuracy</code> considers on how many datapoints the predictions for all output features were correct (but consider that for some features, for instance <code>numeric</code> there is no accuracy measure, so you should use <code>accuracy</code> only if all your output features have an accuracy measure).</li>
<li><code>validation_measure:</code> (default <code>accuracy</code>): the measure to use to determine if there was an improvement. The measure is considered for the output feature specified in <code>validation_field</code>. Different datatypes have different available measures, refer to the datatype-specific section for more details.</li>
<li><code>bucketing_field</code> (default <code>null</code>): when not <code>null</code>, when creating batches, instead of shuffling randomly, the length along the last dimension of the matrix of the specified input feature is used for bucketing datapoints and then randomly shuffled datapoints from the same bin are sampled. Padding is trimmed to the longest datapoint in the batch. The specified feature should be either a <code>sequence</code> or <code>text</code> feature and the encoder encoding it has to be <code>rnn</code>. When used, bucketing improves speed of <code>rnn</code> encoding up to 1.5x, depending on the length distribution of the inputs.</li>
</ul>
<h3 id="preprocessing">Preprocessing<a class="headerlink" href="#preprocessing" title="Permanent link">&para;</a></h3>
<p>The <code>preprocessing</code> section of the model definition makes it possible to specify datatype specific parameters to perform data preprocessing.
The preprocessing dictionary contains one key of each datatype, but you have to specify only the ones that apply to your case, the other ones will be kept as defaults.
Moreover, the preprocessing dictionary contains parameters related to how to split the data that are not feature specific.</p>
<ul>
<li><code>force_split</code> (default <code>false</code>): if <code>true</code> the <code>split</code> column in the CSV data file is ignored and the dataset is randomly split. If <code>false</code> the <code>split</code> column is used if available.</li>
<li><code>split_probabilities</code> (default <code>[0.7, 0.2, 0.1]</code>): the proportion of the CSV data to end up in training, validation and test. The three values have to sum up to one.</li>
<li><code>stratify</code> (default <code>null</code>): if <code>null</code> the split is random, otherwise you can specify the name of a <code>category</code> feature and the split will be stratified on that feature.</li>
</ul>
<p>Example preprocessing dictionary (showing default values):</p>
                <div class="codehilite"><pre><span></span><span
                        class="l l-Scalar l-Scalar-Plain">preprocessing</span><span class="p p-Indicator">:</span>
    <span class="l l-Scalar l-Scalar-Plain">force_split</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
    <span class="l l-Scalar l-Scalar-Plain">split_probabilities</span><span class="p p-Indicator">:</span> <span
                            class="p p-Indicator">[</span><span class="nv">0.7</span><span
                            class="p p-Indicator">,</span> <span class="nv">0.2</span><span
                            class="p p-Indicator">,</span> <span class="nv">0.1</span><span
                            class="p p-Indicator">]</span>
    <span class="l l-Scalar l-Scalar-Plain">stratify</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
    <span class="l l-Scalar l-Scalar-Plain">category</span><span class="p p-Indicator">:</span> <span
                            class="p p-Indicator">{</span><span class="nv">...</span><span
                            class="p p-Indicator">}</span>
    <span class="l l-Scalar l-Scalar-Plain">sequence</span><span class="p p-Indicator">:</span> <span
                            class="p p-Indicator">{</span><span class="nv">...</span><span
                            class="p p-Indicator">}</span>
    <span class="l l-Scalar l-Scalar-Plain">text</span><span class="p p-Indicator">:</span> <span class="p p-Indicator">{</span><span
                            class="nv">...</span><span class="p p-Indicator">}</span>
    <span class="l l-Scalar l-Scalar-Plain">...</span>
</pre></div>


<p>The details about the preprocessing parameters that each datatype accepts will be provided in the datatype-specific sections.</p>
<p>It is important to point out that different features within the same datatype may require different preprocessing.
For instance a document classification model may have two text input features, one for the title of the document and one for the body.</p>
<p>As the length of the title is much shorter than the length of the body, the parameter <code>word_length_limit</code> should be set to 10 for the title and 2000 for the body, but both of them share the same parameter <code>most_common_words</code> with value 10000.</p>
<p>The way to do this is adding a <code>preprocessing</code> key inside the title <code>input_feature</code> dictionary and one in the <code>body</code> input feature dictionary containing the desired parameter and value.
The model definition will look like:</p>
                <div class="codehilite"><pre><span></span><span
                        class="l l-Scalar l-Scalar-Plain">preprocessing</span><span class="p p-Indicator">:</span>
    <span class="l l-Scalar l-Scalar-Plain">text</span><span class="p p-Indicator">:</span>
        <span class="l l-Scalar l-Scalar-Plain">most_common_word</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">10000</span>
<span class="l l-Scalar l-Scalar-Plain">input_features</span><span class="p p-Indicator">:</span>
    <span class="p p-Indicator">-</span>
        <span class="l l-Scalar l-Scalar-Plain">name</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">title</span>
        <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">text</span>
        <span class="l l-Scalar l-Scalar-Plain">preprocessing</span><span class="p p-Indicator">:</span>
            <span class="l l-Scalar l-Scalar-Plain">word_length_limit</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">20</span>
    <span class="p p-Indicator">-</span>
        <span class="l l-Scalar l-Scalar-Plain">name</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">body</span>
        <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">text</span>
        <span class="l l-Scalar l-Scalar-Plain">preprocessing</span><span class="p p-Indicator">:</span>
            <span class="l l-Scalar l-Scalar-Plain">word_length_limit</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">2000</span>
</pre></div>


<h3 id="binary-features">Binary Features<a class="headerlink" href="#binary-features" title="Permanent link">&para;</a></h3>
<h4 id="binary-features-preprocessing">Binary Features Preprocessing<a class="headerlink" href="#binary-features-preprocessing" title="Permanent link">&para;</a></h4>
<p>Binary features are directly transformed into a binary valued vector of length <code>n</code> (where <code>n</code> is the size of the dataset) and added to HDF5 with a key that reflects the name of column in the CSV.
No additional information about them is available in the JSON metadata file.</p>
<p>The parameters available for preprocessing are</p>
<ul>
<li><code>missing_value_strategy</code> (default <code>fill_with_const</code>): what strategy to follow when there's a missing value in a binary column. The value should be one of <code>fill_with_const</code>  (replaces the missing value with a specific value specified with the <code>fill_value</code> parameter), <code>fill_with_mode</code> (replaces the missing values with the most frequent value in the column), <code>fill_with_mean</code> (replaces the missing values with the mean of the values in the column), <code>backfill</code> (replaces the missing values with the next valid value).</li>
<li><code>fill_value</code> (default <code>0</code>): the value to replace the missing values with in case the <code>missing_value_strategy</code> is <code>fill-value</code>.</li>
</ul>
<h4 id="binary-input-features-and-encoders">Binary Input Features and Encoders<a class="headerlink" href="#binary-input-features-and-encoders" title="Permanent link">&para;</a></h4>
<p>Binary features have no encoder, the raw binary values coming from the input placeholders are just returned as outputs.
By consequence there are no encoding parameters.
Inputs are of size <code>b</code> while outputs are fo size <code>b x 1</code> where <code>b</code> is the batch size.</p>
<p>Example binary feature entry in the output features list:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">binary_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">binary</span>
</pre></div>


<h4 id="binary-output-features-and-decoders">Binary Output Features and Decoders<a class="headerlink" href="#binary-output-features-and-decoders" title="Permanent link">&para;</a></h4>
<p>Binary features can be used when a binary classification needs to be performed or when the output is a single probability.
There is only one decoder available for binary features and it is a (potentially empty) stack of fully connected layers, followed by a projection into a single number followed by a sigmoid function.</p>
<p>These are the available parameters of a binary output feature</p>
<ul>
<li><code>reduce_inputs</code> (default <code>sum</code>): defines how to reduce an input that is not a vector, but a matrix or a higher order tensor, on the first dimension 9second if you count the batch dimension). Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension).</li>
<li><code>dependencies</code> (default <code>[]</code>): the output features this one is dependent on. For a detailed explanation refer to <a href="#output-features-dependencies">Output Features Dependencies</a>.</li>
<li><code>reduce_dependencies</code> (default <code>sum</code>): defines how to reduce the output of a dependent feature that is not a vector, but a matrix or a higher order tensor, on the first dimension 9second if you count the batch dimension). Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension).</li>
<li><code>loss</code> (default <code>{type: cross_entropy, confidence_penalty: 0, robust_lambda: 0}</code>): is a dictionary containing a loss <code>type</code> and its hyperparameters. The only available loss <code>type</code> is <code>cross_entropy</code> (cross entropy), and the two optional parameters are <code>confidence_penalty</code> (an additional term that penalizes too confident predictions by adding a <code>a * (max_entropy - entropy) / max_entropy</code> term to the loss, where a is the value of this parameter) and <code>robust_lambda</code> (replaces the loss with <code>(1 - robust_lambda) * loss + robust_lambda / 2</code> which is useful in case of noisy labels).</li>
</ul>
<p>These are the available parameters of a binary output feature decoder</p>
<ul>
<li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the fully connected layers. The length of the list determines the number of stacked fully connected layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>fc_size</code>, <code>norm</code>, <code>activation</code>, <code>dropout</code>, <code>initializer</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the decoder will be used instead.</li>
<li><code>num_fc_layers</code> (default 0): this is the number of stacked fully connected layers that the input to the feature passes through. Their output is projected in the feature's output space.</li>
<li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used for each layer. It indicates the size of the output of a fully connected layer.</li>
<li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not already specified in <code>fc_layers</code> this is the default <code>activation</code> that will be used for each layer. It indicates the activation function applied to the output.</li>
<li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified in <code>fc_layers</code> this is the default <code>norm</code> that will be used for each layer. It indicates the norm of the output and it can be <code>null</code>, <code>batch</code> or <code>layer</code>.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer after each layer.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the wights of the layers are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
<li><code>threshold</code> (default <code>0.5</code>): The threshold above (greater or equal) which the predicted output of the sigmoid will me mapped to 1.</li>
</ul>
<p>Example binary feature entry (with default parameters) in the output features list:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">binary_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">binary</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_inputs</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">dependencies</span><span class="p p-Indicator">:</span> <span
                            class="p p-Indicator">[]</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_dependencies</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">loss</span><span class="p p-Indicator">:</span>
    <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">cross_entropy</span>
    <span class="l l-Scalar l-Scalar-Plain">confidence_penalty</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">robust_lambda</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
<span class="l l-Scalar l-Scalar-Plain">fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
<span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">activation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">relu</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">threshold</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0.5</span>
</pre></div>


<h4 id="binary-features-measures">Binary Features Measures<a class="headerlink" href="#binary-features-measures" title="Permanent link">&para;</a></h4>
<p>The only measures that are calculated every epoch and are available for binary features are the <code>accuracy</code> and the <code>loss</code> itself.
You can set either of them as <code>validation_measure</code> in the <code>training</code> section of the model definition if you set the <code>validation_field</code> to be the name of a binary feature.</p>
<h3 id="numerical-features">Numerical Features<a class="headerlink" href="#numerical-features" title="Permanent link">&para;</a></h3>
<h4 id="numerical-features-preprocessing">Numerical Features Preprocessing<a class="headerlink" href="#numerical-features-preprocessing" title="Permanent link">&para;</a></h4>
<p>Numerical features are directly transformed into a float valued vector of length <code>n</code> (where <code>n</code> is the size of the dataset) and added to HDF5 with a key that reflects the name of column in the CSV.
No additional information about them is available in the JSON metadata file.</p>
<p>Parameters available for preprocessing are</p>
<ul>
<li><code>missing_value_strategy</code> (default <code>fill_with_const</code>): what strategy to follow when there's a missing value in a binary column. The value should be one of <code>fill_with_const</code>  (replaces the missing value with a specific value specified with the <code>fill_value</code> parameter), <code>fill_with_mode</code> (replaces the missing values with the most frequent value in the column), <code>fill_with_mean</code> (replaces the missing values with the mean of the values in the column), <code>backfill</code> (replaces the missing values with the next valid value).</li>
<li><code>fill_value</code> (default <code>0</code>): the value to replace the missing values with in case the <code>missing_value_strategy</code> is <code>fill-value</code>.</li>
</ul>
<h4 id="numerical-input-features-and-encoders">Numerical Input Features and Encoders<a class="headerlink" href="#numerical-input-features-and-encoders" title="Permanent link">&para;</a></h4>
<p>Numerical features have one encoder, the raw float values coming from the input placeholders are passed through a single neuron for scaling purposes, (optionally) passed through a normalization layer (either <code>null</code>, <code>batch_norm</code>, or <code>layer_norm</code>) and returned as outputs.
Inputs are of size <code>b</code> while outputs are fo size <code>b x 1</code> where b is the batch size.</p>
<p>he available encoder parameters arehe available encoder parameters are</p>
<ul>
<li><code>norm'</code> (default <code>null</code>): norm to apply after the single neuron. It can be <code>null</code>, <code>batch</code> or <code>layer</code>.</li>
<li><code>tied_weights</code> (default <code>null</code>): name of the input feature to tie the weights the encoder with. It needs to be the name of a feature of the same type and with the same encoder parameters.</li>
</ul>
<p>Example binary feature entry in the output features list:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">numerical_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">numerical</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">tied_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
</pre></div>


<h4 id="numerical-output-features-and-decoders">Numerical Output Features and Decoders<a class="headerlink" href="#numerical-output-features-and-decoders" title="Permanent link">&para;</a></h4>
<p>Numerical features can be used when a regression needs to be performed.
There is only one decoder available for numerical features and it is a (potentially empty) stack of fully connected layers, followed by a projection into a single number.</p>
<p>These are the available parameters of a numerical output feature</p>
<ul>
<li><code>reduce_inputs</code> (default <code>sum</code>): defines how to reduce an input that is not a vector, but a matrix or a higher order tensor, on the first dimension 9second if you count the batch dimension). Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension).</li>
<li><code>dependencies</code> (default <code>[]</code>): the output features this one is dependent on. For a detailed explanation refer to <a href="#output-features-dependencies">Output Features Dependencies</a>.</li>
<li><code>reduce_dependencies</code> (default <code>sum</code>): defines how to reduce the output of a dependent feature that is not a vector, but a matrix or a higher order tensor, on the first dimension 9second if you count the batch dimension). Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension).</li>
<li><code>loss</code> (default <code>{type: mean_squared_error}</code>): is a dictionary containing a loss <code>type</code>. The available losses <code>type</code> are <code>mean_squared_error</code> and <code>mean_absolute_error</code>.</li>
</ul>
<p>These are the available parameters of a numerical output feature decoder</p>
<ul>
<li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the fully connected layers. The length of the list determines the number of stacked fully connected layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>fc_size</code>, <code>norm</code>, <code>activation</code>, <code>dropout</code>, <code>initializer</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the decoder will be used instead.</li>
<li><code>num_fc_layers</code> (default 0): this is the number of stacked fully connected layers that the input to the feature passes through. Their output is projected in the feature's output space.</li>
<li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used for each layer. It indicates the size of the output of a fully connected layer.</li>
<li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not already specified in <code>fc_layers</code> this is the default <code>activation</code> that will be used for each layer. It indicates the activation function applied to the output.</li>
<li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified in <code>fc_layers</code> this is the default <code>norm</code> that will be used for each layer. It indicates the norm of the output and it can be <code>null</code>, <code>batch</code> or <code>layer</code>.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer after each layer.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the weights of the layers are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
</ul>
<p>Example numerical feature entry (with default parameters) in the output features list:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">numerical_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">numerical</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_inputs</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">dependencies</span><span class="p p-Indicator">:</span> <span
                            class="p p-Indicator">[]</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_dependencies</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">loss</span><span class="p p-Indicator">:</span>
    <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">mean_squared_error</span>
<span class="l l-Scalar l-Scalar-Plain">fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
<span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">activation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">relu</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
</pre></div>


<h4 id="numerical-features-measures">Numerical Features Measures<a class="headerlink" href="#numerical-features-measures" title="Permanent link">&para;</a></h4>
<p>The measures that are calculated every epoch and are available for numerical features are <code>mean_squared_error</code>, <code>mean_absolute_error</code>, <code>r2</code> and the <code>loss</code> itself.
You can set either of them as <code>validation_measure</code> in the <code>training</code> section of the model definition if you set the <code>validation_field</code> to be the name of a binary feature.</p>
<h3 id="category-features">Category Features<a class="headerlink" href="#category-features" title="Permanent link">&para;</a></h3>
<h4 id="category-features-preprocessing">Category Features Preprocessing<a class="headerlink" href="#category-features-preprocessing" title="Permanent link">&para;</a></h4>
<p>Category features are transformed into into an integer valued vector of size <code>n</code> (where <code>n</code> is the size of the dataset) and added to HDF5 with a key that reflects the name of column in the CSV.
The way categories are mapped into integers consists in first collecting a dictionary of all the different category strings present in the column of the CSV, then rank them by frequency and then assign them an increasing integer ID from the most frequent to the most rare (with 0 being assigned to a <code>&lt;UNK&gt;</code> token).
The column name is added to the JSON file, with an associated dictionary containing
1. the mapping from integer to string (<code>idx2str</code>)
2. the mapping from string to id (<code>str2idx</code>)
3. the mapping from string to frequency (<code>str2freq</code>)
4. the size of the set of all tokens (<code>vocab_size</code>)
4. additional preprocessing information (by default how to fill missing values and what token to use to fill missing values)</p>
<p>The parameters available for preprocessing are</p>
<ul>
<li><code>missing_value_strategy</code> (default <code>fill_with_const</code>): what strategy to follow when there's a missing value in a binary column. The value should be one of <code>fill_with_const</code>  (replaces the missing value with a specific value specified with the <code>fill_value</code> parameter), <code>fill_with_mode</code> (replaces the missing values with the most frequent value in the column), <code>fill_with_mean</code> (replaces the missing values with the mean of the values in the column), <code>backfill</code> (replaces the missing values with the next valid value).</li>
<li><code>fill_value</code> (default <code>"&lt;UNK&gt;"</code>): the value to replace the missing values with in case the <code>missing_value_strategy</code> is <code>fill-value</code>.</li>
<li><code>lowercase</code> (default <code>false</code>): if the string has to be lowercased before being handled by the formatter.</li>
<li><code>most_common</code> (default <code>10000</code>): the maximum number of most common tokens to be considered. if the data contains more than this amount, the most infrequent tokens will be treated as unknown.</li>
</ul>
<h4 id="category-input-features-and-encoders">Category Input Features and Encoders<a class="headerlink" href="#category-input-features-and-encoders" title="Permanent link">&para;</a></h4>
<p>Category features have one encoder, the raw integer values coming from the input placeholders are mapped to either dense or sparse embeddings (one-hot encodings) and returned as outputs.
Inputs are of size <code>b</code> while outputs are fo size <code>b x h</code> where <code>b</code> is the batch size and <code>h</code> is the dimensionality of the embeddings.</p>
<p>The available encoder parameters are</p>
<ul>
<li><code>representation'</code> (default <code>dense</code>): the possible values are <code>dense</code> and <code>sparse</code>. <code>dense</code> means the embeddings are initialized randomly, <code>sparse</code> means they are initialized to be one-hot encodings.</li>
<li><code>embedding_size</code> (default <code>256</code>): it is the maximum embedding size, the actual size will be <code>min(vocabulary_size, embedding_size)</code> for <code>dense</code> representations and exactly <code>vocabulary_size</code> for the <code>sparse</code> encoding, where <code>vocabulary_size</code> is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for <code>&lt;UNK&gt;</code>).</li>
<li><code>embeddings_on_cpu</code> (default <code>false</code>): by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory.</li>
<li><code>pretrained_embeddings</code> (default <code>null</code>): by default <code>dense</code> embeddings are initialized randomly, but this parameter allow to specify a path to a file containing embeddings in the <a href="https://nlp.stanford.edu/projects/glove/">GloVe format</a>. When the file containing the embeddings is loaded, only the embeddings with labels present in the vocabulary are kept, the others are discarded. If the vocabulary contains strings that have no match in the embeddings file, their embeddings are initialized with the average of all other embedding plus some random noise to make them different from each other. This parameter has effect only if <code>representation</code> is <code>dense</code>.</li>
<li><code>embeddings_trainable</code> (default <code>true</code>): If <code>true</code> embeddings are trained during the training process, if <code>false</code> embeddings are fixed. It may be useful when loading pretrained embeddings for avoiding finetuning them. This parameter has effect only for <code>representation</code> is <code>dense</code> as <code>sparse</code> one-hot encodings are not trainable.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer after embedding.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the embedding weights are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
<li><code>tied_weights</code> (default <code>null</code>): name of the input feature to tie the weights the encoder with. It needs to be the name of a feature of the same type and with the same encoder parameters.</li>
</ul>
<p>Example category feature entry in the input features list:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">category_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">category</span>
<span class="l l-Scalar l-Scalar-Plain">representation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">dense</span>
<span class="l l-Scalar l-Scalar-Plain">embedding_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_on_cpu</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">pretrained_embeddings</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_trainable</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">tied_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
</pre></div>


<h4 id="category-output-features-and-decoders">Category Output Features and Decoders<a class="headerlink" href="#category-output-features-and-decoders" title="Permanent link">&para;</a></h4>
<p>Category features can be used when a multi-class classification needs to be performed.
There is only one decoder available for category features and it is a (potentially empty) stack of fully connected layers, followed by a projection into a vector of size of the number of available classes, followed by a softmax.</p>
<div class="codehilite"><pre><span></span>+--------------+   +---------+   +-----------+
|Combiner      |   |Fully    |   |Projection |   +-------+
|Output        +---&gt;Connected+---&gt;into Output+---&gt;Softmax|
|Representation|   |Layers   |   |Space      |   +-------+
+--------------+   +---------+   +-----------+
</pre></div>


<p>These are the available parameters of a category output feature</p>
<ul>
<li><code>reduce_inputs</code> (default <code>sum</code>): defines how to reduce an input that is not a vector, but a matrix or a higher order tensor, on the first dimension 9second if you count the batch dimension). Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension).</li>
<li><code>dependencies</code> (default <code>[]</code>): the output features this one is dependent on. For a detailed explanation refer to <a href="#output-features-dependencies">Output Features Dependencies</a>.</li>
<li><code>reduce_dependencies</code> (default <code>sum</code>): defines how to reduce the output of a dependent feature that is not a vector, but a matrix or a higher order tensor, on the first dimension 9second if you count the batch dimension). Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension).</li>
<li><code>loss</code> (default <code>{type: softmax_cross_entropy, class_distance_temperature: 0, class_weights: 1, confidence_penalty: 0, distortion: 1, labels_smoothing: 0, negative_samples: 0, robust_lambda: 0, sampler: null, unique: false}</code>): is a dictionary containing a loss <code>type</code>. The available losses <code>type</code> are <code>softmax_cross_entropy</code> and <code>sampled_softmax_cross_entropy</code>.</li>
</ul>
<p>These are the <code>loss</code> parameters</p>
<ul>
<li><code>confidence_penalty</code> (default <code>0</code>): penalizes overconfident predictions (low entropy) by adding an additional term that penalizes too confident predictions by adding a <code>a * (max_entropy - entropy) / max_entropy</code> term to the loss, where a is the value of this parameter. Useful in case of noisy labels.</li>
<li><code>robust_lambda</code> (default <code>0</code>): replaces the loss with <code>(1 - robust_lambda) * loss + robust_lambda / c</code> where <code>c</code> is the number of classes, which is useful in case of noisy labels.</li>
<li><code>class_weights</code> (default <code>1</code>): the value can be a vector of weights, one of each class, that is multiplied to the loss of the datapoints that have that class as ground truth. It is an alternative to oversampling in case of unbalanced class distribution. The ordering of the vector follows the category to integer ID mapping in the JSON metadata file.</li>
<li><code>class_distances</code> (default <code>null</code>): if not <code>null</code> it is a <code>c x c</code> matrix in the form of a list of lists that contains the mutual similarity of classes. It is used if <code>class_distance_temperature</code> is greater than 0.</li>
<li><code>class_distance_temperature</code> (default <code>0</code>): is the temperature parameter of the softmax_cross_entropy that uses <code>class_weights</code>. The intuition behind it is that errors between similar classes are more tollerable than errors between really different classes.</li>
<li><code>labels_smoothing</code> (default <code>0</code>): If label_smoothing is nonzero, smooth the labels towards <code>1/num_classes</code>: <code>new_onehot_labels = onehot_labels * (1 - label_smoothing) + label_smoothing / num_classes</code>.</li>
<li><code>negative_samples</code> (default <code>0</code>): if <code>type</code> is <code>sampled_softmax_cross_entropy</code>, this parameter indicates how many negative samples to use.</li>
<li><code>sampler</code> (default <code>null</code>): options are <code>fixed_unigram</code>, <code>uniform</code>, <code>log_uniform</code>, <code>learned_unigram</code>. For a detailed description of the samplers refer to <a href="https://www.tensorflow.org/api_guides/python/nn#Candidate_Sampling">TensorFlow's documentation</a>.</li>
<li><code>distortion</code> (default <code>1</code>): when <code>loss</code> is <code>sampled_softmax_cross_entropy</code> and the sampler is either <code>unigram</code> or <code>learned_unigram</code> this is used to skew the unigram probability distribution. Each weight is first raised to the distortion's power before adding to the internal unigram distribution. As a result, distortion = 1.0 gives regular unigram sampling (as defined by the vocab file), and distortion = 0.0 gives a uniform distribution.</li>
<li><code>unique</code> (default <code>false</code>): Determines whether all sampled classes in a batch are unique.</li>
</ul>
<p>These are the available parameters of a category output feature decoder</p>
<ul>
<li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the fully connected layers. The length of the list determines the number of stacked fully connected layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>fc_size</code>, <code>norm</code>, <code>activation</code>, <code>dropout</code>, <code>initializer</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the decoder will be used instead.</li>
<li><code>num_fc_layers</code> (default 0): this is the number of stacked fully connected layers that the input to the feature passes through. Their output is projected in the feature's output space.</li>
<li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used for each layer. It indicates the size of the output of a fully connected layer.</li>
<li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not already specified in <code>fc_layers</code> this is the default <code>activation</code> that will be used for each layer. It indicates the activation function applied to the output.</li>
<li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified in <code>fc_layers</code> this is the default <code>norm</code> that will be used for each layer. It indicates the norm of the output and it can be <code>null</code>, <code>batch</code> or <code>layer</code>.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer after each layer.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the weights of the layers are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
<li><code>top_k</code> (default <code>3</code>): determines the parameter <code>k</code>, the number of categories to consider when computing the <code>top_k</code> measure. It computes accuracy but considering as a match if the true category appears in the first <code>k</code> predicted categories ranked by decoder's confidence.</li>
</ul>
<p>Example category feature entry (with default parameters) in the output features list:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">category_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">category</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_inputs</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">dependencies</span><span class="p p-Indicator">:</span> <span
                            class="p p-Indicator">[]</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_dependencies</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">loss</span><span class="p p-Indicator">:</span>
    <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">softmax_cross_entropy</span>
    <span class="l l-Scalar l-Scalar-Plain">confidence_penalty</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">robust_lambda</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">class_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">1</span>
    <span class="l l-Scalar l-Scalar-Plain">class_distances</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
    <span class="l l-Scalar l-Scalar-Plain">class_distance_temperature</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">labels_smoothing</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">negative_samples</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">sampler</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
    <span class="l l-Scalar l-Scalar-Plain">distortion</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">1</span>
    <span class="l l-Scalar l-Scalar-Plain">unique</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
<span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">activation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">relu</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">top_k</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">3</span>
</pre></div>


<h4 id="category-features-measures">Category Features Measures<a class="headerlink" href="#category-features-measures" title="Permanent link">&para;</a></h4>
<p>The measures that are calculated every epoch and are available for category features are <code>accuracy</code>, <code>top_k</code> (computes accuracy considering as a match if the true category appears in the first <code>k</code> predicted categories ranked by decoder's confidence) and the <code>loss</code> itself.
You can set either of them as <code>validation_measure</code> in the <code>training</code> section of the model definition if you set the <code>validation_field</code> to be the name of a binary feature.</p>
<h3 id="set-features">Set Features<a class="headerlink" href="#set-features" title="Permanent link">&para;</a></h3>
<h4 id="set-features-preprocessing">Set Features Preprocessing<a class="headerlink" href="#set-features-preprocessing" title="Permanent link">&para;</a></h4>
<p>Set features are transformed into into a binary (int8 actually) valued matrix of size <code>n x l</code> (where <code>n</code> is the size of the dataset and <code>l</code> is the minimum of the size of the biggest set and a <code>max_size</code> parameter) and added to HDF5 with a key that reflects the name of column in the CSV.
The way sets are mapped into integers consists in first using a formatter to map from strings to sequences of set items (by default this is done by splitting on spaces).
Then a a dictionary of all the different set item strings present in the column of the CSV is collected, then they are ranked by frequency and an increasing integer ID is assigned to them from the most frequent to the most rare (with 0 being assigned to <code>&lt;PAD&gt;</code> used for padding and 1 assigned to <code>&lt;UNK&gt;</code> item).
The column name is added to the JSON file, with an associated dictionary containing
1. the mapping from integer to string (<code>idx2str</code>)
2. the mapping from string to id (<code>str2idx</code>)
3. the mapping from string to frequency (<code>str2freq</code>)
4. the maximum size of all sets (<code>max_set_size</code>)
5. additional preprocessing information (by default how to fill missing values and what token to use to fill missing values)</p>
<p>The parameters available for preprocessing arehe parameters available for preprocessing are</p>
<ul>
<li><code>missing_value_strategy</code> (default <code>fill_with_const</code>): what strategy to follow when there's a missing value in a binary column. The value should be one of <code>fill_with_const</code>  (replaces the missing value with a specific value specified with the <code>fill_value</code> parameter), <code>fill_with_mode</code> (replaces the missing values with the most frequent value in the column), <code>fill_with_mean</code> (replaces the missing values with the mean of the values in the column), <code>backfill</code> (replaces the missing values with the next valid value).</li>
<li><code>fill_value</code> (default <code>0</code>): the value to replace the missing values with in case the <code>missing_value_strategy</code> is <code>fill-value</code>.</li>
<li><code>format</code> (default <code>space</code>): defines how to map from the raw string content of the CSV column to a set of elements. The default value <code>space</code> splits the string on spaces. Other options are: <code>underscore</code> (splits on underscore), <code>comma</code>(splits on comma), <code>json</code> (decodes the string into a set or a list through a JSON parser).</li>
<li><code>lowercase</code> (default <code>false</code>): if the string has to be lowercased before being handled by the formatter.</li>
<li><code>most_common</code> (default <code>10000</code>): the maximum number of most common tokens to be considered. if the data contains more than this amount, the most infrequent tokens will be treated as unknown.</li>
</ul>
<h4 id="set-input-features-and-encoders">Set Input Features and Encoders<a class="headerlink" href="#set-input-features-and-encoders" title="Permanent link">&para;</a></h4>
<p>Set features have one encoder, the raw binary values coming from the input placeholders are first transformed in sparse integer lists, then they are mapped to either dense or sparse embeddings (one-hot encodings), finally they are aggregated and returned as outputs.
Inputs are of size <code>b</code> while outputs are fo size <code>b x h</code> where <code>b</code> is the batch size and <code>h</code> is the dimensionally of the embeddings.</p>
<div class="codehilite"><pre><span></span>+-+
|0|          +-----+
|0|   +-+    |emb 2|   +-----------+
|1|   |2|    +-----+   |Aggregation|
|0+---&gt;4+----&gt;emb 4+---&gt;Reduce     +-&gt;
|1|   |5|    +-----+   |Operation  |
|1|   +-+    |emb 5|   +-----------+
|0|          +-----+
+-+
</pre></div>


<p>The available encoder parameters are</p>
<ul>
<li><code>representation'</code> (default <code>dense</code>): the possible values are <code>dense</code> and <code>sparse</code>. <code>dense</code> means the embeddings are initialized randomly, <code>sparse</code> means they are initialized to be one-hot encodings.</li>
<li><code>embedding_size</code> (default <code>50</code>): it is the maximum embedding size, the actual size will be <code>min(vocabulary_size, embedding_size)</code> for <code>dense</code> representations and exactly <code>vocabulary_size</code> for the <code>sparse</code> encoding, where <code>vocabulary_size</code> is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for <code>&lt;UNK&gt;</code>).</li>
<li><code>embeddings_on_cpu</code> (default <code>false</code>): by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory.</li>
<li><code>pretrained_embeddings</code> (default <code>null</code>): by default <code>dense</code> embeddings are initialized randomly, but this parameter allow to specify a path to a file containing embeddings in the <a href="https://nlp.stanford.edu/projects/glove/">GloVe format</a>. When the file containing the embeddings is loaded, only the embeddings with labels present in the vocabulary are kept, the others are discarded. If the vocabulary contains strings that have no match in the embeddings file, their embeddings are initialized with the average of all other embedding plus some random noise to make them different from each other. This parameter has effect only if <code>representation</code> is <code>dense</code>.</li>
<li><code>embeddings_trainable</code> (default <code>true</code>): If <code>true</code> embeddings are trained during the training process, if <code>false</code> embeddings are fixed. It may be useful when loading pretrained embeddings for avoiding finetuning them. This parameter has effect only for <code>representation</code> is <code>dense</code> as <code>sparse</code> one-hot encodings are not trainable.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer before returning the encoder output.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the embedding weights are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
<li><code>reduce_output</code> (default <code>sum</code>): describes the strategy to use to aggregate the embeddings of the items of the set. Possible values are <code>sum</code>, <code>mean</code> and <code>sqrt</code> (the weighted sum divided by the square root of the sum of the squares of the weights).</li>
<li><code>tied_weights</code> (default <code>null</code>): name of the input feature to tie the weights the encoder with. It needs to be the name of a feature of the same type and with the same encoder parameters.</li>
</ul>
<p>Example set feature entry in the output features list:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">set_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">set</span>
<span class="l l-Scalar l-Scalar-Plain">representation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">dense</span>
<span class="l l-Scalar l-Scalar-Plain">embedding_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">50</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_on_cpu</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">pretrained_embeddings</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_trainable</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_output</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">tied_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
</pre></div>


<h4 id="set-output-features-and-decoders">Set Output Features and Decoders<a class="headerlink" href="#set-output-features-and-decoders" title="Permanent link">&para;</a></h4>
<p>Set features can be used when multi-label classification needs to be performed.
There is only one decoder available for set features and it is a (potentially empty) stack of fully connected layers, followed by a projection into a vector of size of the number of available classes, followed by a sigmoid.</p>
<div class="codehilite"><pre><span></span>+--------------+   +---------+   +-----------+
|Combiner      |   |Fully    |   |Projection |   +-------+
|Output        +---&gt;Connected+---&gt;into Output+---&gt;Sigmoid|
|Representation|   |Layers   |   |Space      |   +-------+
+--------------+   +---------+   +-----------+
</pre></div>


<p>These are the available parameters of the set output feature</p>
<ul>
<li><code>reduce_inputs</code> (default <code>sum</code>): defines how to reduce an input that is not a vector, but a matrix or a higher order tensor, on the first dimension 9second if you count the batch dimension). Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension).</li>
<li><code>dependencies</code> (default <code>[]</code>): the output features this one is dependent on. For a detailed explanation refer to <a href="#output-features-dependencies">Output Features Dependencies</a>.</li>
<li><code>reduce_dependencies</code> (default <code>sum</code>): defines how to reduce the output of a dependent feature that is not a vector, but a matrix or a higher order tensor, on the first dimension 9second if you count the batch dimension). Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension).</li>
<li><code>loss</code> (default <code>{type: sigmoid_cross_entropy}</code>): is a dictionary containing a loss <code>type</code>. The available loss <code>type</code> is <code>sigmoid_cross_entropy</code>.</li>
</ul>
<p>These are the available parameters of a set output feature decoder</p>
<ul>
<li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the fully connected layers. The length of the list determines the number of stacked fully connected layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>fc_size</code>, <code>norm</code>, <code>activation</code>, <code>dropout</code>, <code>initializer</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the decoder will be used instead.</li>
<li><code>num_fc_layers</code> (default 0): this is the number of stacked fully connected layers that the input to the feature passes through. Their output is projected in the feature's output space.</li>
<li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used for each layer. It indicates the size of the output of a fully connected layer.</li>
<li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not already specified in <code>fc_layers</code> this is the default <code>activation</code> that will be used for each layer. It indicates the activation function applied to the output.</li>
<li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified in <code>fc_layers</code> this is the default <code>norm</code> that will be used for each layer. It indicates the norm of the output and it can be <code>null</code>, <code>batch</code> or <code>layer</code>.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer after each layer.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the wights of the layers are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
<li><code>threshold</code> (default <code>0.5</code>): The threshold above (greater or equal) which the predicted output of the sigmoid will me mapped to 1.</li>
</ul>
<p>Example set feature entry (with default parameters) in the output features list:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">set_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">set</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_inputs</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">dependencies</span><span class="p p-Indicator">:</span> <span
                            class="p p-Indicator">[]</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_dependencies</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">loss</span><span class="p p-Indicator">:</span>
    <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sigmoid_cross_entropy</span>
<span class="l l-Scalar l-Scalar-Plain">fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
<span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">activation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">relu</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">threshold</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0.5</span>
</pre></div>


<h4 id="set-features-measures">Set Features Measures<a class="headerlink" href="#set-features-measures" title="Permanent link">&para;</a></h4>
<p>The measures that are calculated every epoch and are available for category features are <code>jaccard_index</code> and the <code>loss</code> itself.
You can set either of them as <code>validation_measure</code> in the <code>training</code> section of the model definition if you set the <code>validation_field</code> to be the name of a binary feature.</p>
<h3 id="bag-features">Bag Features<a class="headerlink" href="#bag-features" title="Permanent link">&para;</a></h3>
<h4 id="bag-features-preprocessing">Bag Features Preprocessing<a class="headerlink" href="#bag-features-preprocessing" title="Permanent link">&para;</a></h4>
<p>Bag features are treated in the same way of set features, with the only difference being that the matrix had float values (frequencies).</p>
<h4 id="bag-input-features-and-encoders">Bag Input Features and Encoders<a class="headerlink" href="#bag-input-features-and-encoders" title="Permanent link">&para;</a></h4>
<p>Bag features have one encoder, the raw float values coming from the input placeholders are first transformed in sparse integer lists, then they are mapped to either dense or sparse embeddings (one-hot encodings), they are aggregated as a weighted sum, where the weights are the original float values, and finally returned as outputs.
Inputs are of size <code>b</code> while outputs are fo size <code>b x h</code> where <code>b</code> is the batch size and <code>h</code> is the dimensionality of the embeddings.</p>
<p>The parameters are the same used for set input features with the exception of <code>reduce_output</code> that doesn't apply in this case because the weighted sum already acts as a reducer.</p>
<h4 id="bag-output-features-and-decoders">Bag Output Features and Decoders<a class="headerlink" href="#bag-output-features-and-decoders" title="Permanent link">&para;</a></h4>
<p>There is no bag decoder available yet.</p>
<h4 id="bag-features-measures">Bag Features Measures<a class="headerlink" href="#bag-features-measures" title="Permanent link">&para;</a></h4>
<p>As there is no decoder there is also no measure available yet for bag feature.</p>
<h3 id="sequence-features">Sequence Features<a class="headerlink" href="#sequence-features" title="Permanent link">&para;</a></h3>
<h4 id="sequence-features-preprocessing">Sequence Features Preprocessing<a class="headerlink" href="#sequence-features-preprocessing" title="Permanent link">&para;</a></h4>
<p>Sequence features are transformed into into an integer valued matrix of size <code>n x l</code> (where <code>n</code> is the size of the dataset and <code>l</code> is the minimum of the length of the longest sequence and a <code>sequence_length_limit</code> parameter) and added to HDF5 with a key that reflects the name of column in the CSV.
The way sets are mapped into integers consists in first using a formatter to map from strings to sequences of tokens (by default this is done by splitting on spaces).
Then a a dictionary of all the different token strings present in the column of the CSV is collected, then they are ranked by frequency and an increasing integer ID is assigned to them from the most frequent to the most rare (with 0 being assigned to <code>&lt;PAD&gt;</code> used for padding and 1 assigned to <code>&lt;UNK&gt;</code> item).
The column name is added to the JSON file, with an associated dictionary containing
1. the mapping from integer to string (<code>idx2str</code>)
2. the mapping from string to id (<code>str2idx</code>)
3. the mapping from string to frequency (<code>str2freq</code>)
4. the maximum length of all sequences (<code>sequence_length_limit</code>)
5. additional preprocessing information (by default how to fill missing values and what token to use to fill missing values)</p>
<p>The parameters available for preprocessing are</p>
<ul>
<li><code>missing_value_strategy</code> (default <code>fill_with_const</code>): what strategy to follow when there's a missing value in a binary column. The value should be one of <code>fill_with_const</code>  (replaces the missing value with a specific value specified with the <code>fill_value</code> parameter), <code>fill_with_mode</code> (replaces the missing values with the most frequent value in the column), <code>fill_with_mean</code> (replaces the missing values with the mean of the values in the column), <code>backfill</code> (replaces the missing values with the next valid value).</li>
<li><code>fill_value</code> (default <code>""</code>): the value to replace the missing values with in case the <code>missing_value_strategy</code> is <code>fill_value</code>.</li>
<li><code>padding</code> (default <code>right</code>): the direction of the padding. <code>right</code> and <code>left</code> are available options.</li>
<li><code>padding_symbol</code> (default <code>&lt;PAD&gt;</code>): the string used as a padding symbol. Is is mapped to the integer ID 0 in the vocabulary.</li>
<li><code>unknown_symbol</code> (default <code>&lt;UNK&gt;</code>): the string used as a unknown symbol. Is is mapped to the integer ID 1 in the vocabulary.</li>
<li><code>lowercase</code> (default <code>false</code>): if the string has to be lowercase before being handled by the formatter.</li>
<li><code>format</code> (default <code>space</code>): defines how to map from the raw string content of the CSV column to a sequence of elements. The default value <code>space</code> splits the string on spaces. Other options are: <code>underscore</code> (splits on underscore), <code>comma</code>(splits on comma), <code>json</code> (decodes the string into a set or a list through a JSON parser).</li>
<li><code>most_common</code> (default <code>20000</code>): the maximum number of most common tokens to be considered. if the data contains more than this amount, the most infrequent tokens will be treated as unknown.</li>
<li><code>sequence_length_limit</code> (default <code>256</code>): the maximum length of the sequence. Sequences that are longer than this value will be truncated, while sequences that are shorter will be padded.</li>
</ul>
<h4 id="sequence-input-features-and-encoders">Sequence Input Features and Encoders<a class="headerlink" href="#sequence-input-features-and-encoders" title="Permanent link">&para;</a></h4>
<p>Sequence features have several encoders and each of them has its own parameters.
Inputs are of size <code>b</code> while outputs are fo size <code>b x h</code> where <code>b</code> is the batch size and <code>h</code> is the dimensionally of the output of the encoder.
In case a representation for each element of the sequence is needed (for example for tagging them, or for using an attention mechanism), one can specify the parameter <code>reduce_output</code> to be <code>null</code> or <code>null</code> and the output will be a <code>b x s x h</code> tensor where <code>s</code> is the length of the sequence.
Some encoders, because of their inner workings, may require additional parameters to be specified in order to obtain one representation for each element of the sequence.
For instance the <code>parallel_cnn</code> encoder, by default pools and flattens the sequence dimension and then passes the flattened vector through fully connected layers, so in order to obtain the full tesnor one has to specify <code>reduce_output: null</code>.</p>
<p>Sequence input feature parameters are</p>
<ul>
<li><code>encoder</code> (default <code>parallel_cnn</code>): the name of the encoder to use to encode the sequence. The available ones are  <code>embed</code>, <code>parallel_cnn</code>, <code>stacked_cnn</code>, <code>stacked_parallel_cnn</code>, <code>rnn</code> and <code>cnnrnn</code>.</li>
<li><code>tied_weights</code> (default <code>null</code>): name of the input feature to tie the weights the encoder with. It needs to be the name of a feature of the same type and with the same encoder parameters.</li>
</ul>
<h5 id="embed-encoder">Embed Encoder<a class="headerlink" href="#embed-encoder" title="Permanent link">&para;</a></h5>
<p>The embed decoder simply maps each integer in the sequence to an embedding, creating a <code>b x s x h</code> tensor where <code>b</code> is the batch size, <code>s</code> is the length of the sequence and <code>h</code> is the embedding size.
The tensor is reduced along the <code>s</code> dimension to obtain a single vector of size <code>h</code> for each element of the batch.
If you want to output the full <code>b x s x h</code> tensor, you can specify <code>reduce_output: null</code>.</p>
<div class="codehilite"><pre><span></span>       +------+
       |Emb 12|
       +------+
+--+   |Emb 7 |
|12|   +------+
|7 |   |Emb 43|   +-----------+
|43|   +------+   |Aggregation|
|65+---&gt;Emb 65+---&gt;Reduce     +-&gt;
|23|   +------+   |Operation  |
|4 |   |Emb 23|   +-----------+
|1 |   +------+
+--+   |Emb 4 |
       +------+
       |Emb 1 |
       +------+
</pre></div>


<p>These are the parameters available for the embed encoder</p>
<ul>
<li><code>representation'</code> (default <code>dense</code>): the possible values are <code>dense</code> and <code>sparse</code>. <code>dense</code> means the embeddings are initialized randomly, <code>sparse</code> means they are initialized to be one-hot encodings.</li>
<li><code>embedding_size</code> (default <code>50</code>): it is the maximum embedding size, the actual size will be <code>min(vocabulary_size, embedding_size)</code> for <code>dense</code> representations and exactly <code>vocabulary_size</code> for the <code>sparse</code> encoding, where <code>vocabulary_size</code> is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for <code>&lt;UNK&gt;</code>).</li>
<li><code>embeddings_on_cpu</code> (default <code>false</code>): by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory.</li>
<li><code>pretrained_embeddings</code> (default <code>null</code>): by default <code>dense</code> embeddings are initialized randomly, but this parameter allow to specify a path to a file containing embeddings in the <a href="https://nlp.stanford.edu/projects/glove/">GloVe format</a>. When the file containing the embeddings is loaded, only the embeddings with labels present in the vocabulary are kept, the others are discarded. If the vocabulary contains strings that have no match in the embeddings file, their embeddings are initialized with the average of all other embedding plus some random noise to make them different from each other. This parameter has effect only if <code>representation</code> is <code>dense</code>.</li>
<li><code>embeddings_trainable</code> (default <code>true</code>): If <code>true</code> embeddings are trained during the training process, if <code>false</code> embeddings are fixed. It may be useful when loading pretrained embeddings for avoiding finetuning them. This parameter has effect only for <code>representation</code> is <code>dense</code> as <code>sparse</code> one-hot encodings are not trainable.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer before returning the encoder output.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the embedding weights are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
<li><code>reduce_output</code> (default <code>sum</code>): defines how to reduce the output tensor along the <code>s</code> sequence length dimension if the rank of the tensor is greater than 2. Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension) and <code>null</code> or <code>null</code> (which does not reduce and returns the full tensor).</li>
</ul>
<p>Example sequence feature entry in the output features list using an embed encoder:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">sequence_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sequence</span>
<span class="l l-Scalar l-Scalar-Plain">encoder</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">parallel_cnn</span>
<span class="l l-Scalar l-Scalar-Plain">tied_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">representation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">dense</span>
<span class="l l-Scalar l-Scalar-Plain">embedding_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_on_cpu</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">pretrained_embeddings</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_trainable</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_output</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
</pre></div>


<h5 id="parallel-cnn-encoder">Parallel CNN Encoder<a class="headerlink" href="#parallel-cnn-encoder" title="Permanent link">&para;</a></h5>
<p>The parallel cnn encoder is inspired by <a href="https://arxiv.org/abs/1408.5882">Yoon Kim's Convolutional Neural Network for Sentence Classification</a>.
It works by first mapping the input integer sequence <code>b x s</code> (where <code>b</code> is the batch size and <code>s</code> is the length of the sequence) into a sequence of embeddings, then it passes the embedding through a number of parallel 1d convolutional layers with different filter size (by default 4 layers with filter size 2, 3, 4 and 5), followed by max pooling and concatenation.
This single vector concatenating the outputs of the parallel convolutional layers is then passed through a stack of fully connected layers and returned as a <code>b x h</code> tensor where <code>h</code> is the output size of the last fully connected layer.
If you want to output the full <code>b x s x h</code> tensor, you can specify <code>reduce_output: null</code>.</p>
<div class="codehilite"><pre><span></span>                   +-------+   +----+
                +--&gt;1D Conv+---&gt;Pool+-+
       +------+ |  |Width 2|   +----+ |
       |Emb 12| |  +-------+          |
       +------+ |                     |
+--+   |Emb 7 | |  +-------+   +----+ |
|12|   +------+ +--&gt;1D Conv+---&gt;Pool+-+
|7 |   |Emb 43| |  |Width 3|   +----+ |           +---------+
|43|   +------+ |  +-------+          | +------+  |Fully    |
|65+---&gt;Emb 65+-+                     +-&gt;Concat+--&gt;Connected+-&gt;
|23|   +------+ |  +-------+   +----+ | +------+  |Layers   |
|4 |   |Emb 23| +--&gt;1D Conv+---&gt;Pool+-+           +---------+
|1 |   +------+ |  |Width 4|   +----+ |
+--+   |Emb 4 | |  +-------+          |
       +------+ |                     |
       |Emb 1 | |  +-------+   +----+ |
       +------+ +--&gt;1D Conv+---&gt;Pool+-+
                   |Width 5|   +----+
                   +-------+
</pre></div>


<p>These are the available for an parallel cnn encoder:</p>
<ul>
<li><code>representation'</code> (default <code>dense</code>): the possible values are <code>dense</code> and <code>sparse</code>. <code>dense</code> means the embeddings are initialized randomly, <code>sparse</code> means they are initialized to be one-hot encodings.</li>
<li><code>embedding_size</code> (default <code>256</code>): it is the maximum embedding size, the actual size will be <code>min(vocabulary_size, embedding_size)</code> for <code>dense</code> representations and exactly <code>vocabulary_size</code> for the <code>sparse</code> encoding, where <code>vocabulary_size</code> is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for <code>&lt;UNK&gt;</code>).</li>
<li><code>embeddings_on_cpu</code> (default <code>false</code>): by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory.</li>
<li><code>pretrained_embeddings</code> (default <code>null</code>): by default <code>dense</code> embeddings are initialized randomly, but this parameter allow to specify a path to a file containing embeddings in the <a href="https://nlp.stanford.edu/projects/glove/">GloVe format</a>. When the file containing the embeddings is loaded, only the embeddings with labels present in the vocabulary are kept, the others are discarded. If the vocabulary contains strings that have no match in the embeddings file, their embeddings are initialized with the average of all other embedding plus some random noise to make them different from each other. This parameter has effect only if <code>representation</code> is <code>dense</code>.</li>
<li><code>embeddings_trainable</code> (default <code>true</code>): If <code>true</code> embeddings are trained during the training process, if <code>false</code> embeddings are fixed. It may be useful when loading pretrained embeddings for avoiding finetuning them. This parameter has effect only for <code>representation</code> is <code>dense</code> as <code>sparse</code> one-hot encodings are not trainable.</li>
<li><code>conv_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the convolutional layers. The length of the list determines the number of parallel convolutional layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>filter_size</code>, <code>num_filters</code>, <code>pool</code>, <code>norm</code>, <code>activation</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the encoder will be used instead. If both <code>conv_layers</code> and <code>num_conv_layers</code> are <code>null</code>, a default list will be assigned to <code>conv_layers</code> with the value <code>[{filter_size: 2}, {filter_size: 3}, {filter_size: 4}, {filter_size: 5}]</code>.</li>
<li><code>num_conv_layers</code> (default <code>null</code>): if <code>conv_layers</code> is <code>null</code>, this is the number of parallel convolutional layers.</li>
<li><code>filter_size</code> (default <code>3</code>): if a <code>filter_size</code> is not already specified in <code>conv_layers</code> this is the default <code>filter_size</code> that will be used for each layer. It indicates how wide is the 1d convolutional filter.</li>
<li><code>num_filters</code> (default <code>256</code>): if a <code>num_filters</code> is not already specified in <code>conv_layers</code> this is the default <code>num_filters</code> that will be used for each layer. It indicates the number of filters, and by consequence the output channels of the 1d convolution.</li>
<li><code>pool_size</code> (default <code>null</code>): if a <code>pool_size</code> is not already specified in <code>conv_layers</code> this is the default <code>pool_size</code> that will be used for each layer. It indicates the size of the max pooling that will be performed along the <code>s</code> sequence dimension after the convolution operation.</li>
<li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the fully connected layers. The length of the list determines the number of stacked fully connected layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>fc_size</code>, <code>norm</code>, <code>activation</code>,  <code>initializer</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the encoder will be used instead. If both <code>fc_layers</code> and <code>num_fc_layers</code> are <code>null</code>, a default list will be assigned to <code>fc_layers</code> with the value <code>[{fc_size: 512}, {fc_size: 256}]</code>. (only applies if <code>reduce_output</code> is not <code>null</code>).</li>
<li><code>num_fc_layers</code> (default <code>null</code>): if <code>fc_layers</code> is <code>null</code>, this is the number of stacked fully connected layers (only applies if <code>reduce_output</code> is not <code>null</code>).</li>
<li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used for each layer. It indicates the size of the output of a fully connected layer.</li>
<li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not already specified in <code>conv_layers</code> or <code>fc_layers</code> this is the default <code>activation</code> that will be used for each layer. It indicates the activation function applied to the output.</li>
<li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified in <code>conv_layers</code> or <code>fc_layers</code> this is the default <code>norm</code> that will be used for each layer. It indicates the norm of the output.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer after each layer.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code> it uses <code>glorot_uniform</code>. Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if a <code>regularize</code> is not already specified in <code>conv_layers</code> or <code>fc_layers</code> this is the default <code>regularize</code> that will be used for each layer. It indicates if the layer weights should be considered when computing a regularization loss.</li>
<li><code>reduce_output</code> (default <code>sum</code>): defines how to reduce the output tensor along the <code>s</code> sequence length dimension if the rank of the tensor is greater than 2. Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the sequence dimension), <code>last</code> (returns the last vector of the sequence dimension) and <code>null</code> or <code>null</code> (which does not reduce and returns the full tensor).</li>
</ul>
<p>Example sequence feature entry in the output features list using a parallel cnn encoder:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">sequence_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sequence</span>
<span class="l l-Scalar l-Scalar-Plain">encoder</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">parallel_cnn</span>
<span class="l l-Scalar l-Scalar-Plain">tied_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">representation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">dense</span>
<span class="l l-Scalar l-Scalar-Plain">embedding_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_on_cpu</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">pretrained_embeddings</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_trainable</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">conv_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_conv_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">filter_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">3</span>
<span class="l l-Scalar l-Scalar-Plain">num_filters</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">pool_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">activation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">relu</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_output sum</span>
</pre></div>


<h5 id="stacked-cnn-encoder">Stacked CNN Encoder<a class="headerlink" href="#stacked-cnn-encoder" title="Permanent link">&para;</a></h5>
<p>The stacked cnn encoder is inspired by <a href="https://arxiv.org/abs/1509.01626">Xiang Zhang at all's Character-level Convolutional Networks for Text Classification</a>.
It works by first mapping the input integer sequence <code>b x s</code> (where <code>b</code> is the batch size and <code>s</code> is the length of the sequence) into a sequence of embeddings, then it passes the embedding through a stack of 1d convolutional layers with different filter size (by default 6 layers with filter size 7, 7, 3, 3, 3 and 3), followed by an optional final pool and by a flatten operation.
This single flatten vector is then passed through a stack of fully connected layers and returned as a <code>b x h</code> tensor where <code>h</code> is the output size of the last fully connected layer.
If you want to output the full <code>b x s x h</code> tensor, you can specify the <code>pool_size</code> of all your <code>conv_layers</code> to be <code>null</code>  and <code>reduce_output: null</code>, while if <code>pool_size</code> has a value different from <code>null</code> and <code>reduce_output: null</code> the returned tensor will be of shape <code>b x s' x h</code>, where <code>s'</code> is width of the output of the last convolutional layer.</p>
<div class="codehilite"><pre><span></span>       +------+
       |Emb 12|
       +------+
+--+   |Emb 7 |
|12|   +------+
|7 |   |Emb 43|   +----------------+  +---------+
|43|   +------+   |1D Conv         |  |Fully    |
|65+---&gt;Emb 65+---&gt;Layers          +--&gt;Connected+-&gt;
|23|   +------+   |Different Widths|  |Layers   |
|4 |   |Emb 23|   +----------------+  +---------+
|1 |   +------+
+--+   |Emb 4 |
       +------+
       |Emb 1 |
       +------+
</pre></div>


<p>These are the parameters available for the stack cnn encoder:</p>
<ul>
<li><code>representation'</code> (default <code>dense</code>): the possible values are <code>dense</code> and <code>sparse</code>. <code>dense</code> means the embeddings are initialized randomly, <code>sparse</code> means they are initialized to be one-hot encodings.</li>
<li><code>embedding_size</code> (default <code>256</code>): it is the maximum embedding size, the actual size will be <code>min(vocabulary_size, embedding_size)</code> for <code>dense</code> representations and exactly <code>vocabulary_size</code> for the <code>sparse</code> encoding, where <code>vocabulary_size</code> is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for <code>&lt;UNK&gt;</code>).</li>
<li><code>embeddings_on_cpu</code> (default <code>false</code>): by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory.</li>
<li><code>pretrained_embeddings</code> (default <code>null</code>): by default <code>dense</code> embeddings are initialized randomly, but this parameter allow to specify a path to a file containing embeddings in the <a href="https://nlp.stanford.edu/projects/glove/">GloVe format</a>. When the file containing the embeddings is loaded, only the embeddings with labels present in the vocabulary are kept, the others are discarded. If the vocabulary contains strings that have no match in the embeddings file, their embeddings are initialized with the average of all other embedding plus some random noise to make them different from each other. This parameter has effect only if <code>representation</code> is <code>dense</code>.</li>
<li><code>embeddings_trainable</code> (default <code>true</code>): If <code>true</code> embeddings are trained during the training process, if <code>false</code> embeddings are fixed. It may be useful when loading pretrained embeddings for avoiding finetuning them. This parameter has effect only for <code>representation</code> is <code>dense</code> as <code>sparse</code> one-hot encodings are not trainable.</li>
<li><code>conv_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the convolutional layers. The length of the list determines the number of stacked convolutional layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>filter_size</code>, <code>num_filters</code>, <code>pool_size</code>, <code>norm</code>, <code>activation</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the encoder will be used instead. If both <code>conv_layers</code> and <code>num_conv_layers</code> are <code>null</code>, a default list will be assigned to <code>conv_layers</code> with the value <code>[{filter_size: 7, pool_size: 3, regularize: false}, {filter_size: 7, pool_size: 3, regularize: false}, {filter_size: 3, pool_size: null, regularize: false}, {filter_size: 3, pool_size: null, regularize: false}, {filter_size: 3, pool_size: null, regularize: true}, {filter_size: 3, pool_size: 3, regularize: true}]</code>.</li>
<li><code>num_conv_layers</code> (default <code>null</code>): if <code>conv_layers</code> is <code>null</code>, this is the number of stacked convolutional layers.</li>
<li><code>filter_size</code> (default <code>3</code>): if a <code>filter_size</code> is not already specified in <code>conv_layers</code> this is the default <code>filter_size</code> that will be used for each layer. It indicates how wide is the 1d convolutional filter.</li>
<li><code>num_filters</code> (default <code>256</code>): if a <code>num_filters</code> is not already specified in <code>conv_layers</code> this is the default <code>num_filters</code> that will be used for each layer. It indicates the number of filters, and by consequence the output channels of the 1d convolution.</li>
<li><code>pool_size</code> (default <code>null</code>): if a <code>pool_size</code> is not already specified in <code>conv_layers</code> this is the default <code>pool_size</code> that will be used for each layer. It indicates the size of the max pooling that will be performed along the <code>s</code> sequence dimension after the convolution operation.</li>
<li><code>reduce_output</code> (default <code>max</code>): defines how to reduce the output tensor of the convolutional layers along the <code>s</code> sequence length dimension if the rank of the tensor is greater than 2. Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension) and <code>null</code> or <code>null</code> (which does not reduce and returns the full tensor).</li>
<li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the fully connected layers. The length of the list determines the number of stacked fully connected layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>fc_size</code>, <code>norm</code>, <code>activation</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the encoder will be used instead. If both <code>fc_layers</code> and <code>num_fc_layers</code> are <code>null</code>, a default list will be assigned to <code>fc_layers</code> with the value <code>[{fc_size: 512}, {fc_size: 256}]</code>. (only applies if <code>reduce_output</code> is not <code>null</code>).</li>
<li><code>num_fc_layers</code> (default <code>null</code>): if <code>fc_layers</code> is <code>null</code>, this is the number of stacked fully connected layers (only applies if <code>reduce_output</code> is not <code>null</code>).</li>
<li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used for each layer. It indicates the size of the output of a fully connected layer.</li>
<li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not already specified in <code>conv_layers</code> or <code>fc_layers</code> this is the default <code>activation</code> that will be used for each layer. It indicates the activation function applied to the output.</li>
<li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified in <code>conv_layers</code> or <code>fc_layers</code> this is the default <code>norm</code> that will be used for each layer. It indicates the norm of the output.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer after each layer.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code> it uses <code>glorot_uniform</code>. Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if a <code>regularize</code> is not already specified in <code>conv_layers</code> or <code>fc_layers</code> this is the default <code>regularize</code> that will be used for each layer. It indicates if the layer weights should be considered when computing a regularization loss.</li>
<li><code>reduce_output</code> (default <code>sum</code>): defines how to reduce the output tensor along the <code>s</code> sequence length dimension if the rank of the tensor is greater than 2. Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension) and <code>null</code> or <code>null</code> (which does not reduce and returns the full tensor).</li>
</ul>
<p>Example sequence feature entry in the output features list using a parallel cnn encoder:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">sequence_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sequence</span>
<span class="l l-Scalar l-Scalar-Plain">encoder</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">stacked_cnn</span>
<span class="l l-Scalar l-Scalar-Plain">tied_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">representation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">dense</span>
<span class="l l-Scalar l-Scalar-Plain">embedding_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_on_cpu</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">pretrained_embeddings</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_trainable</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">conv_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_conv_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">filter_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">3</span>
<span class="l l-Scalar l-Scalar-Plain">num_filters</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">pool_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">activation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">relu</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_output</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">max</span>
</pre></div>


<h5 id="stacked-parallel-cnn-encoder">Stacked Parallel CNN Encoder<a class="headerlink" href="#stacked-parallel-cnn-encoder" title="Permanent link">&para;</a></h5>
<p>The stacked parallel cnn encoder is a combination of the Parallel CNN and the Stacked CNN encoders where each layer of the stack is a composed of parallel convolutional layers.
It works by first mapping the input integer sequence <code>b x s</code> (where <code>b</code> is the batch size and <code>s</code> is the length of the sequence) into a sequence of embeddings, then it passes the embedding through a stack of several parallel 1d convolutional layers with different filter size, followed by an optional final pool and by a flatten operation.
This single flatten vector is then passed through a stack of fully connected layers and returned as a <code>b x h</code> tensor where <code>h</code> is the output size of the last fully connected layer.
If you want to output the full <code>b x s x h</code> tensor, you can specify <code>reduce_output: null</code>.</p>
<div class="codehilite"><pre><span></span>                   +-------+                      +-------+
                +--&gt;1D Conv+-+                 +--&gt;1D Conv+-+
       +------+ |  |Width 2| |                 |  |Width 2| |
       |Emb 12| |  +-------+ |                 |  +-------+ |
       +------+ |            |                 |            |
+--+   |Emb 7 | |  +-------+ |                 |  +-------+ |
|12|   +------+ +--&gt;1D Conv+-+                 +--&gt;1D Conv+-+
|7 |   |Emb 43| |  |Width 3| |                 |  |Width 3| |                   +---------+
|43|   +------+ |  +-------+ | +------+  +---+ |  +-------+ | +------+  +----+  |Fully    |
|65+---&gt;Emb 65+-+            +-&gt;Concat+--&gt;...+-+            +-&gt;Concat+--&gt;Pool+--&gt;Connected+-&gt;
|23|   +------+ |  +-------+ | +------+  +---+ |  +-------+ | +------+  +----+  |Layers   |
|4 |   |Emb 23| +--&gt;1D Conv+-+                 +--&gt;1D Conv+-+                   +---------+
|1 |   +------+ |  |Width 4| |                 |  |Width 4| |
+--+   |Emb 4 | |  +-------+ |                 |  +-------+ |
       +------+ |            |                 |            |
       |Emb 1 | |  +-------+ |                 |  +-------+ |
       +------+ +--&gt;1D Conv+-+                 +--&gt;1D Conv+-+
                   |Width 5|                      |Width 5|
                   +-------+                      +-------+
</pre></div>


<p>These are the available parameters for the stack parallel cnn encoder:</p>
<ul>
<li><code>representation'</code> (default <code>dense</code>): the possible values are <code>dense</code> and <code>sparse</code>. <code>dense</code> means the embeddings are initialized randomly, <code>sparse</code> means they are initialized to be one-hot encodings.</li>
<li><code>embedding_size</code> (default <code>256</code>): it is the maximum embedding size, the actual size will be <code>min(vocabulary_size, embedding_size)</code> for <code>dense</code> representations and exactly <code>vocabulary_size</code> for the <code>sparse</code> encoding, where <code>vocabulary_size</code> is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for <code>&lt;UNK&gt;</code>).</li>
<li><code>embeddings_on_cpu</code> (default <code>false</code>): by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory.</li>
<li><code>pretrained_embeddings</code> (default <code>null</code>): by default <code>dense</code> embeddings are initialized randomly, but this parameter allow to specify a path to a file containing embeddings in the <a href="https://nlp.stanford.edu/projects/glove/">GloVe format</a>. When the file containing the embeddings is loaded, only the embeddings with labels present in the vocabulary are kept, the others are discarded. If the vocabulary contains strings that have no match in the embeddings file, their embeddings are initialized with the average of all other embedding plus some random noise to make them different from each other. This parameter has effect only if <code>representation</code> is <code>dense</code>.</li>
<li><code>embeddings_trainable</code> (default <code>true</code>): If <code>true</code> embeddings are trained during the training process, if <code>false</code> embeddings are fixed. It may be useful when loading pretrained embeddings for avoiding finetuning them. This parameter has effect only for <code>representation</code> is <code>dense</code> as <code>sparse</code> one-hot encodings are not trainable.</li>
<li><code>stacked_layers</code> (default <code>null</code>): it is a of lists of list of dictionaries containing the parameters of the stack of parallel convolutional layers. The length of the list determines the number of stacked parallel convolutional layers, length of the sub-lists determines the number of parallel conv layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>filter_size</code>, <code>num_filters</code>, <code>pool_size</code>, <code>norm</code>, <code>activation</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the encoder will be used instead. If both <code>stacked_layers</code> and <code>num_stacked_layers</code> are <code>null</code>, a default list will be assigned to <code>stacked_layers</code> with the value <code>[[{filter_size: 2}, {filter_size: 3}, {filter_size: 4}, {filter_size: 5}], [{filter_size: 2}, {filter_size: 3}, {filter_size: 4}, {filter_size: 5}], [{filter_size: 2}, {filter_size: 3}, {filter_size: 4}, {filter_size: 5}]]</code>.</li>
<li><code>num_stacked_layers</code> (default <code>null</code>): if <code>stacked_layers</code> is <code>null</code>, this is the number of elements in the stack of parallel convolutional layers.</li>
<li><code>filter_size</code> (default <code>3</code>): if a <code>filter_size</code> is not already specified in <code>conv_layers</code> this is the default <code>filter_size</code> that will be used for each layer. It indicates how wide is the 1d convolutional filter.</li>
<li><code>num_filters</code> (default <code>256</code>): if a <code>num_filters</code> is not already specified in <code>conv_layers</code> this is the default <code>num_filters</code> that will be used for each layer. It indicates the number of filters, and by consequence the output channels of the 1d convolution.</li>
<li><code>pool_size</code> (default <code>null</code>): if a <code>pool_size</code> is not already specified in <code>conv_layers</code> this is the default <code>pool_size</code> that will be used for each layer. It indicates the size of the max pooling that will be performed along the <code>s</code> sequence dimension after the convolution operation.</li>
<li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the fully connected layers. The length of the list determines the number of stacked fully connected layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>fc_size</code>, <code>norm</code>, <code>activation</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the encoder will be used instead. If both <code>fc_layers</code> and <code>num_fc_layers</code> are <code>null</code>, a default list will be assigned to <code>fc_layers</code> with the value <code>[{fc_size: 512}, {fc_size: 256}]</code>. (only applies if <code>reduce_output</code> is not <code>null</code>).</li>
<li><code>num_fc_layers</code> (default <code>null</code>): if <code>fc_layers</code> is <code>null</code>, this is the number of stacked fully connected layers (only applies if <code>reduce_output</code> is not <code>null</code>).</li>
<li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used for each layer. It indicates the size of the output of a fully connected layer.</li>
<li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified in <code>conv_layers</code> or <code>fc_layers</code> this is the default <code>norm</code> that will be used for each layer. It indicates the norm of the output.</li>
<li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not already specified in <code>conv_layers</code> or <code>fc_layers</code> this is the default <code>activation</code> that will be used for each layer. It indicates the activation function applied to the output.</li>
<li><code>regularize</code> (default <code>true</code>): if a <code>regularize</code> is not already specified in <code>conv_layers</code> or <code>fc_layers</code> this is the default <code>regularize</code> that will be used for each layer. It indicates if the layer weights should be considered when computing a regularization loss.</li>
<li><code>reduce_output</code> (default <code>sum</code>): defines how to reduce the output tensor along the <code>s</code> sequence length dimension if the rank of the tensor is greater than 2. Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension) and <code>null</code> or <code>null</code> (which does not reduce and returns the full tensor).</li>
</ul>
<p>Example sequence feature entry in the output features list using a parallel cnn encoder:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">sequence_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sequence</span>
<span class="l l-Scalar l-Scalar-Plain">encoder</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">stacked_parallel_cnn</span>
<span class="l l-Scalar l-Scalar-Plain">tied_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">representation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">dense</span>
<span class="l l-Scalar l-Scalar-Plain">embedding_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_on_cpu</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">pretrained_embeddings</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_trainable</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">stacked_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_stacked_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">filter_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">3</span>
<span class="l l-Scalar l-Scalar-Plain">num_filters</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">pool_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">activation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">relu</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_output</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">max</span>
</pre></div>


<h5 id="rnn-encoder">RNN Encoder<a class="headerlink" href="#rnn-encoder" title="Permanent link">&para;</a></h5>
<p>The rnn encoder works by first mapping the input integer sequence <code>b x s</code> (where <code>b</code> is the batch size and <code>s</code> is the length of the sequence) into a sequence of embeddings, then it passes the embedding through a stack of recurrent layers (by default 1 layer), followed by a reduce operation that by default only returns the last output, but can perform other reduce functions.
If you want to output the full <code>b x s x h</code> where <code>h</code> is the size of the output of the last rnn layer, you can specify <code>reduce_output: null</code>.</p>
<div class="codehilite"><pre><span></span>       +------+
       |Emb 12|
       +------+
+--+   |Emb 7 |
|12|   +------+
|7 |   |Emb 43|                 +---------+
|43|   +------+   +----------+  |Fully    |
|65+---&gt;Emb 65+---&gt;RNN Layers+--&gt;Connected+-&gt;
|23|   +------+   +----------+  |Layers   |
|4 |   |Emb 23|                 +---------+
|1 |   +------+
+--+   |Emb 4 |
       +------+
       |Emb 1 |
       +------+
</pre></div>


<p>These are the available parameters for the rnn encoder:</p>
<ul>
<li><code>representation'</code> (default <code>dense</code>): the possible values are <code>dense</code> and <code>sparse</code>. <code>dense</code> means the embeddings are initialized randomly, <code>sparse</code> means they are initialized to be one-hot encodings.</li>
<li><code>embedding_size</code> (default <code>256</code>): it is the maximum embedding size, the actual size will be <code>min(vocabulary_size, embedding_size)</code> for <code>dense</code> representations and exactly <code>vocabulary_size</code> for the <code>sparse</code> encoding, where <code>vocabulary_size</code> is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for <code>&lt;UNK&gt;</code>).</li>
<li><code>embeddings_on_cpu</code> (default <code>false</code>): by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory.</li>
<li><code>pretrained_embeddings</code> (default <code>null</code>): by default <code>dense</code> embeddings are initialized randomly, but this parameter allow to specify a path to a file containing embeddings in the <a href="https://nlp.stanford.edu/projects/glove/">GloVe format</a>. When the file containing the embeddings is loaded, only the embeddings with labels present in the vocabulary are kept, the others are discarded. If the vocabulary contains strings that have no match in the embeddings file, their embeddings are initialized with the average of all other embedding plus some random noise to make them different from each other. This parameter has effect only if <code>representation</code> is <code>dense</code>.</li>
<li><code>embeddings_trainable</code> (default <code>true</code>): If <code>true</code> embeddings are trained during the training process, if <code>false</code> embeddings are fixed. It may be useful when loading pretrained embeddings for avoiding finetuning them. This parameter has effect only for <code>representation</code> is <code>dense</code> as <code>sparse</code> one-hot encodings are not trainable.</li>
<li><code>num_layers</code> (default <code>1</code>): the number of stacked recurrent layers.</li>
<li><code>cell_type</code> (default <code>rnn</code>): the type of recurrent cell to use. Available values are: <code>rnn</code>, <code>lstm</code>, <code>lstm_block</code>, <code>lstm</code>, <code>ln</code>, <code>lstm_cudnn</code>, <code>gru</code>, <code>gru_block</code>, <code>gru_cudnn</code>. For reference about the differences between the cells please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell">TensorFlow's documentation</a>. We suggest to use the <code>block</code> variants on CPU and the <code>cudnn</code> variants on GPU because of their increased speed.</li>
<li><code>state_size</code> (default <code>256</code>): the size of the state of the rnn.</li>
<li><code>bidirectional</code> (default <code>false</code>): if <code>true</code> two recurrent networks will perform encoding in the forward and backward direction and their outputs will be concatenated.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer before returning the encoder output.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the embedding weights are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
<li><code>reduce_output</code> (default <code>last</code>): defines how to reduce the output tensor along the <code>s</code> sequence length dimension if the rank of the tensor is greater than 2. Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension) and <code>null</code> or <code>null</code> (which does not reduce and returns the full tensor).</li>
</ul>
<p>Example sequence feature entry in the output features list using a parallel cnn encoder:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">sequence_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sequence</span>
<span class="l l-Scalar l-Scalar-Plain">encoder</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">rnn</span>
<span class="l l-Scalar l-Scalar-Plain">tied_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">representation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">dense</span>
<span class="l l-Scalar l-Scalar-Plain">embedding_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_on_cpu</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">pretrained_embeddings</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_trainable</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">num_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">1</span>
<span class="l l-Scalar l-Scalar-Plain">cell_type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">rnn</span>
<span class="l l-Scalar l-Scalar-Plain">state_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">bidirectional</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_output sum</span>
</pre></div>


<h5 id="cnn-rnn-encoder">CNN RNN Encoder<a class="headerlink" href="#cnn-rnn-encoder" title="Permanent link">&para;</a></h5>
<p>The cnn rnn encoder works by first mapping the input integer sequence <code>b x s</code> (where <code>b</code> is the batch size and <code>s</code> is the length of the sequence) into a sequence of embeddings, then it passes the embedding through a stack of convolutional layers (by default 2), that is followed by a stack of recurrent layers (by default 1), followed by a reduce operation that by default only returns the last output, but can perform other reduce functions.
If you want to output the full <code>b x s x h</code> where <code>h</code> is the size of the output of the last rnn layer, you can specify <code>reduce_output: null</code>.</p>
<div class="codehilite"><pre><span></span>       +------+
       |Emb 12|
       +------+
+--+   |Emb 7 |
|12|   +------+
|7 |   |Emb 43|                                +---------+
|43|   +------+   +----------+   +----------+  |Fully    |
|65+---&gt;Emb 65+---&gt;CNN Layers+---&gt;RNN Layers+--&gt;Connected+-&gt;
|23|   +------+   +----------+   +----------+  |Layers   |
|4 |   |Emb 23|                                +---------+
|1 |   +------+
+--+   |Emb 4 |
       +------+
       |Emb 1 |
       +------+
</pre></div>


<p>These are the available parameters of the cnn rnn encoder:</p>
<ul>
<li><code>representation'</code> (default <code>dense</code>): the possible values are <code>dense</code> and <code>sparse</code>. <code>dense</code> means the embeddings are initialized randomly, <code>sparse</code> means they are initialized to be one-hot encodings.</li>
<li><code>embedding_size</code> (default <code>256</code>): it is the maximum embedding size, the actual size will be <code>min(vocabulary_size, embedding_size)</code> for <code>dense</code> representations and exactly <code>vocabulary_size</code> for the <code>sparse</code> encoding, where <code>vocabulary_size</code> is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for <code>&lt;UNK&gt;</code>).</li>
<li><code>embeddings_on_cpu</code> (default <code>false</code>): by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory.</li>
<li><code>pretrained_embeddings</code> (default <code>null</code>): by default <code>dense</code> embeddings are initialized randomly, but this parameter allow to specify a path to a file containing embeddings in the <a href="https://nlp.stanford.edu/projects/glove/">GloVe format</a>. When the file containing the embeddings is loaded, only the embeddings with labels present in the vocabulary are kept, the others are discarded. If the vocabulary contains strings that have no match in the embeddings file, their embeddings are initialized with the average of all other embedding plus some random noise to make them different from each other. This parameter has effect only if <code>representation</code> is <code>dense</code>.</li>
<li><code>embeddings_trainable</code> (default <code>true</code>): If <code>true</code> embeddings are trained during the training process, if <code>false</code> embeddings are fixed. It may be useful when loading pretrained embeddings for avoiding finetuning them. This parameter has effect only for <code>representation</code> is <code>dense</code> as <code>sparse</code> one-hot encodings are not trainable.</li>
<li><code>conv_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the convolutional layers. The length of the list determines the number of stacked convolutional layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>filter_size</code>, <code>num_filters</code>, <code>pool_size</code>, <code>norm</code>, <code>activation</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the encoder will be used instead. If both <code>conv_layers</code> and <code>num_conv_layers</code> are <code>null</code>, a default list will be assigned to <code>conv_layers</code> with the value <code>[{filter_size: 7, pool_size: 3, regularize: false}, {filter_size: 7, pool_size: 3, regularize: false}, {filter_size: 3, pool_size: null, regularize: false}, {filter_size: 3, pool_size: null, regularize: false}, {filter_size: 3, pool_size: null, regularize: true}, {filter_size: 3, pool_size: 3, regularize: true}]</code>.</li>
<li><code>num_conv_layers</code> (default <code>null</code>): if <code>conv_layers</code> is <code>null</code>, this is the number of parallel convolutional layers.</li>
<li><code>filter_size</code> (default <code>3</code>): if a <code>filter_size</code> is not already specified in <code>conv_layers</code> this is the default <code>filter_size</code> that will be used for each layer. It indicates how wide is the 1d convolutional filter.</li>
<li><code>num_filters</code> (default <code>256</code>): if a <code>num_filters</code> is not already specified in <code>conv_layers</code> this is the default <code>num_filters</code> that will be used for each layer. It indicates the number of filters, and by consequence the output channels of the 1d convolution.</li>
<li><code>pool_size</code> (default <code>null</code>): if a <code>pool_size</code> is not already specified in <code>conv_layers</code> this is the default <code>pool_size</code> that will be used for each layer. It indicates the size of the max pooling that will be performed along the <code>s</code> sequence dimension after the convolution operation.</li>
<li><code>num_rec_layers</code> (default <code>1</code>): the number of stacked recurrent layers.</li>
<li><code>cell_type</code> (default <code>rnn</code>): the type of recurrent cell to use. Available values are: <code>rnn</code>, <code>lstm</code>, <code>lstm_block</code>, <code>lstm</code>, <code>ln</code>, <code>lstm_cudnn</code>, <code>gru</code>, <code>gru_block</code>, <code>gru_cudnn</code>. For reference about the differences between the cells please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell">TensorFlow's documentstion</a>. We suggest to use the <code>block</code> variants on CPU and the <code>cudnn</code> variants on GPU because of their increased speed.</li>
<li><code>state_size</code> (default <code>256</code>): the size of the state of the rnn.</li>
<li><code>bidirectional</code> (default <code>false</code>): if <code>true</code> two recurrent networks will perform encoding in the forward and backward direction and their outputs will be concatenated.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer between <code>conv_layers</code> and before returning the encoder output.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the embedding weights are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
<li><code>reduce_output</code> (default <code>last</code>): defines how to reduce the output tensor along the <code>s</code> sequence length dimension if the rank of the tensor is greater than 2. Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension) and <code>null</code> or <code>null</code> (which does not reduce and returns the full tensor).</li>
</ul>
<p>Example sequence feature entry in the output features list using a parallel cnn encoder:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">sequence_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sequence</span>
<span class="l l-Scalar l-Scalar-Plain">encoder</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">rnn</span>
<span class="l l-Scalar l-Scalar-Plain">tied_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">representation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">dense</span>
<span class="l l-Scalar l-Scalar-Plain">embedding_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_on_cpu</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">pretrained_embeddings</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">embeddings_trainable</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">conv_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_conv_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">filter_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">3</span>
<span class="l l-Scalar l-Scalar-Plain">num_filters</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">pool_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">activation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">relu</span>
<span class="l l-Scalar l-Scalar-Plain">num_rec_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">1</span>
<span class="l l-Scalar l-Scalar-Plain">cell_type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">rnn</span>
<span class="l l-Scalar l-Scalar-Plain">state_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">bidirectional</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_output</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">last</span>
</pre></div>


<h4 id="sequence-output-features-and-decoders">Sequence Output Features and Decoders<a class="headerlink" href="#sequence-output-features-and-decoders" title="Permanent link">&para;</a></h4>
<p>Sequential features can be used when sequence tagging (classifying each element of an input sequence) or sequence generation needs to be performed.
There are two decoders available for those to tasks names <code>tagger</code> and <code>generator</code>.</p>
<p>These are the available parameters of a sequence output feature</p>
<ul>
<li><code>reduce_inputs</code> (default <code>sum</code>): defines how to reduce an input that is not a vector, but a matrix or a higher order tensor, on the first dimension 9second if you count the batch dimension). Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension).</li>
<li><code>dependencies</code> (default <code>[]</code>): the output features this one is dependent on. For a detailed explanation refer to <a href="#output-features-dependencies">Output Features Dependencies</a>.</li>
<li><code>reduce_dependencies</code> (default <code>sum</code>): defines how to reduce the output of a dependent feature that is not a vector, but a matrix or a higher order tensor, on the first dimension 9second if you count the batch dimension). Available values are: <code>sum</code>, <code>mean</code> or <code>avg</code>, <code>max</code>, <code>concat</code> (concatenates along the first dimension), <code>last</code> (returns the last vector of the first dimension).</li>
<li><code>loss</code> (default <code>{type: softmax_cross_entropy, class_distance_temperature: 0, class_weights: 1, confidence_penalty: 0, distortion: 1, labels_smoothing: 0, negative_samples: 0, robust_lambda: 0, sampler: null, unique: false}</code>): is a dictionary containing a loss <code>type</code>. The available losses <code>type</code> are <code>softmax_cross_entropy</code> and <code>sampled_softmax_cross_entropy</code>. For details on both losses, please refer to the <a href="#category-output-features-and-encoders">category feature output feature section</a>.</li>
</ul>
<h5 id="tagger-decoder">Tagger Decoder<a class="headerlink" href="#tagger-decoder" title="Permanent link">&para;</a></h5>
<p>In the case of <code>tagger</code> the decoder is a (potentially empty) stack of fully connected layers, followed by a projection into a tensor of size <code>b x s x c</code>, where <code>b</code> is the batch size, <code>s</code> is the length of the sequence and <code>c</code> is the number of classes, followed by a softmax_cross_entropy.
This decoder requires its input to be shaped as <code>b x s x h</code>, where <code>h</code> is an hidden dimension, which is the output of a sequence, text or timeseries input feature without reduced outputs or the output of a sequence-based combiner.
If a <code>b x h</code> input is provided instead, an error will be raised during model building.</p>
<div class="codehilite"><pre><span></span>Combiner
Output

+---+                 +----------+   +-------+
|emb|   +---------+   |Projection|   |Softmax|
+---+   |Fully    |   +----------+   +-------+
|...+---&gt;Connected+---&gt;...       +---&gt;...    |
+---+   |Layers   |   +----------+   +-------+
|emb|   +---------+   |Projection|   |Softmax|
+---+                 +----------+   +-------+
</pre></div>


<p>These are the available parameters of a tagger decoder:</p>
<ul>
<li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the fully connected layers. The length of the list determines the number of stacked fully connected layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>fc_size</code>, <code>norm</code>, <code>activation</code>, <code>dropout</code>, <code>initializer</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the decoder will be used instead.</li>
<li><code>num_fc_layers</code> (default 0): this is the number of stacked fully connected layers that the input to the feature passes through. Their output is projected in the feature's output space.</li>
<li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used for each layer. It indicates the size of the output of a fully connected layer.</li>
<li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not already specified in <code>fc_layers</code> this is the default <code>activation</code> that will be used for each layer. It indicates the activation function applied to the output.</li>
<li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified in <code>fc_layers</code> this is the default <code>norm</code> that will be used for each layer. It indicates the norm of the output and it can be <code>null</code>, <code>batch</code> or <code>layer</code>.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer after each layer.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the weights of the layers are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
</ul>
<p>Example sequence feature entry using a tagger decoder (with default parameters) in the output features list:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">sequence_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sequence</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_inputs</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">dependencies</span><span class="p p-Indicator">:</span> <span
                            class="p p-Indicator">[]</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_dependencies</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">loss</span><span class="p p-Indicator">:</span>
    <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">softmax_cross_entropy</span>
    <span class="l l-Scalar l-Scalar-Plain">confidence_penalty</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">robust_lambda</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">class_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">1</span>
    <span class="l l-Scalar l-Scalar-Plain">class_distances</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
    <span class="l l-Scalar l-Scalar-Plain">class_distance_temperature</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">labels_smoothing</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">negative_samples</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">sampler</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
    <span class="l l-Scalar l-Scalar-Plain">distortion</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">1</span>
    <span class="l l-Scalar l-Scalar-Plain">unique</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
<span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">activation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">relu</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
</pre></div>


<h5 id="generator-decoder">Generator Decoder<a class="headerlink" href="#generator-decoder" title="Permanent link">&para;</a></h5>
<p>In the case of <code>generator</code> the decoder is a (potentially empty) stack of fully connected layers, followed by an rnn that generates outputs feeding on its own previous predictions and generates a tensor of size <code>b x s' x c</code>, where <code>b</code> is the batch size, <code>s'</code> is the length of the generated sequence and <code>c</code> is the number of classes, followed by a softmax_cross_entropy.
By default a generator expects a <code>b x h</code> shaped input tensor, where <code>h</code> is a hidden dimension.
The <code>h</code> vectors are (after an optional stack of fully connected layers) fed into the rnn generator.
One exception is when the generator uses attention, as in that case the expected size of the input tensor is <code>b x s x h</code>, which is the output of a sequence, text or timeseries input feature without reduced outputs or the output of a sequence-based combiner.
If a <code>b x h</code> input is provided to a generator decoder using an rnn with attention instead, an error will be raised during model building.</p>
<div class="codehilite"><pre><span></span>                            Output     Output
                               1  +-+    ... +--+    END
                               ^    |     ^     |     ^
+--------+   +---------+       |    |     |     |     |
|Combiner|   |Fully    |   +---+--+ | +---+---+ | +---+--+
|Output  +---&gt;Connected+---+RNN   +---&gt;RNN... +---&gt;RNN   |
|        |   |Layers   |   +---^--+ | +---^---+ | +---^--+
+--------+   +---------+       |    |     |     |     |
                              GO    +-----+     +-----+
</pre></div>


<p>These are the available parameters of a tagger decoder:</p>
<ul>
<li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the fully connected layers. The length of the list determines the number of stacked fully connected layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>fc_size</code>, <code>norm</code>, <code>activation</code>, <code>dropout</code>, <code>initializer</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the decoder will be used instead.</li>
<li><code>num_fc_layers</code> (default 0): this is the number of stacked fully connected layers that the input to the feature passes through. Their output is projected in the feature's output space.</li>
<li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used for each layer. It indicates the size of the output of a fully connected layer.</li>
<li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not already specified in <code>fc_layers</code> this is the default <code>activation</code> that will be used for each layer. It indicates the activation function applied to the output.</li>
<li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified in <code>fc_layers</code> this is the default <code>norm</code> that will be used for each layer. It indicates the norm of the output and it can be <code>null</code>, <code>batch</code> or <code>layer</code>.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer after each layer.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the weights of the layers are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
<li><code>cell_type</code> (default <code>rnn</code>): the type of recurrent cell to use. Available values are: <code>rnn</code>, <code>lstm</code>, <code>lstm_block</code>, <code>lstm</code>, <code>ln</code>, <code>lstm_cudnn</code>, <code>gru</code>, <code>gru_block</code>, <code>gru_cudnn</code>. For reference about the differences between the cells please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell">TensorFlow's documentstion</a>. We suggest to use the <code>block</code> variants on CPU and the <code>cudnn</code> variants on GPU because of their increased speed.</li>
<li><code>state_size</code> (default <code>256</code>): the size of the state of the rnn.</li>
<li><code>tied_embeddings</code> (default <code>null</code>): if <code>null</code> the embeddings of the targets are initialized randomly, while if the values is the name of an input feature, the embeddings of that input feature will be used as embeddings of the target. The <code>vocabulary_size</code> of that input feature has to be the same of the output feature one and it has to have an embedding matrix (binary and numerical features will not have one, fo instance). In this case the <code>embedding_size</code> will be the same as the <code>state_size</code>. This is useful for implementing autoencoders where the encoding and decoding part of the model share parameters.</li>
<li><code>embedding_size</code> (default 256): if <code>tied_target_embeddings</code> is <code>false</code>, the input embeddings and the weights of the softmax_cross_entropy weights before the softmax_cross_entropy are not tied together and can have different sizes, this parameter describes the size of the embeddings of the inputs of the generator.</li>
<li><code>beam_width</code> (default <code>1</code>): sampling from the rnn generator is performed using beam search. By default, with a beam of one, only a greedy sequence using always the most probably next token is generated, but the beam size can be increased. This usually leads to better performance at the expense of more computation and slower generation.</li>
<li><code>attention_mechanism</code> (default <code>null</code>): the recurrent generator may use an attention mechanism. The available ones are <code>bahdanau</code> and <code>luong</code> (for more information refer to <a href="https://www.tensorflow.org/api_guides/python/contrib.seq2seq#Attention">TensorFlow's documentation</a>). When <code>attention</code> is not <code>null</code> the expected size of the input tensor is <code>b x s x h</code>, which is the output of a sequence, text or timeseries input feature without reduced outputs or the output of a sequence-based combiner. If a <code>b x h</code> input is provided to a generator decoder using an rnn with attention instead, an error will be raised during model building.</li>
</ul>
<p>Example sequence feature entry using a tagger decoder (with default parameters) in the output features list:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">name</span><span
                        class="p p-Indicator">:</span> <span
                        class="l l-Scalar l-Scalar-Plain">sequence_csv_column_name</span>
<span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sequence</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_inputs</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">dependencies</span><span class="p p-Indicator">:</span> <span
                            class="p p-Indicator">[]</span>
<span class="l l-Scalar l-Scalar-Plain">reduce_dependencies</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">sum</span>
<span class="l l-Scalar l-Scalar-Plain">loss</span><span class="p p-Indicator">:</span>
    <span class="l l-Scalar l-Scalar-Plain">type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">softmax_cross_entropy</span>
    <span class="l l-Scalar l-Scalar-Plain">confidence_penalty</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">robust_lambda</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">class_weights</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">1</span>
    <span class="l l-Scalar l-Scalar-Plain">class_distances</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
    <span class="l l-Scalar l-Scalar-Plain">class_distance_temperature</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">labels_smoothing</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">negative_samples</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
    <span class="l l-Scalar l-Scalar-Plain">sampler</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
    <span class="l l-Scalar l-Scalar-Plain">distortion</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">1</span>
    <span class="l l-Scalar l-Scalar-Plain">unique</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
<span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">activation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">relu</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">cell_type</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">rnn</span>
<span class="l l-Scalar l-Scalar-Plain">state_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">tied_target_embeddings</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
<span class="l l-Scalar l-Scalar-Plain">embedding_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">beam_width</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">1</span>
<span class="l l-Scalar l-Scalar-Plain">attention_mechanism</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
</pre></div>


<h4 id="sequence-features-measures">Sequence Features Measures<a class="headerlink" href="#sequence-features-measures" title="Permanent link">&para;</a></h4>
<p>The measures that are calculated every epoch and are available for category features are <code>accuracy</code> (counts the number of datapoints where all the elements of the predicted sequence are correct over the number of all datapoints), <code>overall_accuracy</code> (computes the number of elements in all the sequences that are correctly predicted over the number of all the elements in all the sequences), <code>last_accuracy</code> (accuracy considering only the last element of the sequence, it is useful for being sure special end-of-sequence tokens are generated or tagged), <code>edit_distance</code> (the levenshtein distance between the predicted and ground truth sequence), <code>perplexity</code> (the perplexity of the ground truth sequence according to the model) and the <code>loss</code> itself.
You can set either of them as <code>validation_measure</code> in the <code>training</code> section of the model definition if you set the <code>validation_field</code> to be the name of a binary feature.</p>
<h3 id="text-features">Text Features<a class="headerlink" href="#text-features" title="Permanent link">&para;</a></h3>
<h4 id="text-features-preprocessing">Text Features Preprocessing<a class="headerlink" href="#text-features-preprocessing" title="Permanent link">&para;</a></h4>
<p>Text features are treated in the same way of sequence features, with a couple differences.
Two different formattings/splittings happen, one that splits at every character and one that uses a spaCy based tokenizer (and removes stopwords) are used, and two different key are added to the HDF5 file, one containing the matrix of characters and one containing the matrix of words.
The same thing happens in the JSON file, where there are dictionaries for mapping characters to integers (and the inverse) and words to integers (and their inverse).
In the model definition you are able to specify which level of representation to use, if the character level or the word level.</p>
<p>The parameters available for preprocessing are:</p>
<ul>
<li><code>missing_value_strategy</code> (default <code>fill_with_const</code>): what strategy to follow when there's a missing value in a binary column. The value should be one of <code>fill_with_const</code>  (replaces the missing value with a specific value specified with the <code>fill_value</code> parameter), <code>fill_with_mode</code> (replaces the missing values with the most frequent value in the column), <code>fill_with_mean</code> (replaces the missing values with the mean of the values in the column), <code>backfill</code> (replaces the missing values with the next valid value).</li>
<li><code>fill_value</code> (default <code>""</code>): the value to replace the missing values with in case the <code>missing_value_strategy</code> is <code>fill-value</code>.</li>
<li><code>padding</code> (default <code>right</code>): the direction of the padding. <code>right</code> and <code>left</code> are available options.</li>
<li><code>padding_symbol</code> (default <code>&lt;PAD&gt;</code>): the string used as a padding symbol. Is is mapped to the integer ID 0 in the vocabulary.</li>
<li><code>unknown_symbol</code> (default <code>&lt;UNK&gt;</code>): the string used as a unknown symbol. Is is mapped to the integer ID 1 in the vocabulary.</li>
<li><code>lowercase</code> (default <code>false</code>): if the string has to be lowercased before being handled by the formatter.</li>
<li><code>word_sequence_length_limit</code> (default <code>256</code>): the maximum length of the text in words. Texts that are longer than this value will be truncated, while texts that are shorter will be padded.</li>
<li><code>format_words</code> (default <code>space_punct</code>): defines how to map from the raw string content of the CSV column to a sequence of words. The default value <code>space_punct</code> splits the string using a regular expression that separates also punctuation. Other options are: <code>space</code> (splits on space), <code>underscore</code> (splits on underscore), <code>comma</code>(splits on comma), <code>json</code> (decodes the string into a set or a list through a JSON parser), and a set of format functions that rely on <a href="https://spacy.io">spaCy</a>. The spaCy based ones are: <code>english_tokenize</code> (uses spaCy tokenizer), <code>english_tokenize_filter</code> (uses spaCy tokenizer and filters out punctuation, numbers, stopwords and words shorter than 3 characters), <code>english_tokenize_remove_stopwords</code> (uses spaCy tokenizer and filters out stopwords), <code>english_lemmatize</code> (uses spaCy lemmatizer), <code>english_lemmatize_filter</code> (uses spaCy lemmatizer and filters out punctuation, numbers, stopwords and words shorter than 3 characters), <code>english_lemmatize_remove_stopwords</code> (uses spaCy lemmatize and filters out stopwords).</li>
<li><code>most_common_words</code> (default <code>20000</code>): the maximum number of most common words to be considered. If the data contains more than this amount, the most infrequent words will be treated as unknown.</li>
<li><code>char_sequence_length_limit</code> (default <code>1024</code>): the maximum length of the text in characters. Texts that are longer than this value will be truncated, while sequences that are shorter will be padded.</li>
<li><code>format_characters</code> (default <code>characters</code>): defines how to map from the raw string content of the CSV column to a sequence of characters. The default value and only available option is <code>characters</code> and the behavior is to split the string at each character.</li>
<li><code>most_common_characters</code> (default <code>70</code>): the maximum number of most common characters to be considered. if the data contains more than this amount, the most infrequent characters will be treated as unknown.</li>
</ul>
<h4 id="text-input-features-and-encoders">Text Input Features and Encoders<a class="headerlink" href="#text-input-features-and-encoders" title="Permanent link">&para;</a></h4>
<p>The encoders are the same used for the <a href="#sequence-input-features-and-encoders">Sequence Features</a>.
The only difference is that you can specify an additional <code>level</code> parameter with possible values <code>word</code> or <code>char</code> to force to use the text words or characters as inputs (by default the encoder will sue <code>word</code>).</p>
<h4 id="text-output-features-and-decoders">Text Output Features and Decoders<a class="headerlink" href="#text-output-features-and-decoders" title="Permanent link">&para;</a></h4>
<p>The decoders are the same used for the <a href="#sequence-output-features-and-decoders">Sequence Features</a>.
The only difference is that you can specify an additional <code>level</code> parameter with possible values <code>word</code> or <code>char</code> to force to use the text words or characters as inputs (by default the encoder will sue <code>word</code>).</p>
<h4 id="text-features-measures">Text Features Measures<a class="headerlink" href="#text-features-measures" title="Permanent link">&para;</a></h4>
<p>The measures are the same used for the <a href="#sequence-features-measures">Sequence Features</a>.</p>
<h3 id="time-series-features">Time Series Features<a class="headerlink" href="#time-series-features" title="Permanent link">&para;</a></h3>
<h4 id="time-series-features-preprocessing">Time Series Features Preprocessing<a class="headerlink" href="#time-series-features-preprocessing" title="Permanent link">&para;</a></h4>
<p>Timeseries features are treated in the same way of sequence features, with the only difference being that the matrix in the HDF5 file does not have integer values, but float values.
Moreover, there is no need for any mapping in the JSON file.</p>
<h4 id="time-series-input-features-and-encoders">Time Series Input Features and Encoders<a class="headerlink" href="#time-series-input-features-and-encoders" title="Permanent link">&para;</a></h4>
<p>The encoders are the same used for the <a href="#sequence-input-features-and-encoders">Sequence Features</a>.
The only difference is that time series features don't have an embedding layer at the beginning, so the <code>b x s</code> placeholders (where <code>b</code> is the batch size and <code>s</code> is the sequence length) are directly mapped to a <code>b x s x 1</code> tensor and then passed to the different sequential encoders.</p>
<h4 id="time-series-output-features-and-decoders">Time Series Output Features and Decoders<a class="headerlink" href="#time-series-output-features-and-decoders" title="Permanent link">&para;</a></h4>
<p>There are no time series decoders at the moment (WIP), so time series cannot be used as output features.</p>
<h4 id="time-series-features-measures">Time Series Features Measures<a class="headerlink" href="#time-series-features-measures" title="Permanent link">&para;</a></h4>
<p>As no time series decoders are available at the moment, there are also no time series measures.</p>
<h3 id="image-features">Image Features<a class="headerlink" href="#image-features" title="Permanent link">&para;</a></h3>
<h4 id="image-features-preprocessing">Image Features Preprocessing<a class="headerlink" href="#image-features-preprocessing" title="Permanent link">&para;</a></h4>
<p>Ludwig supports grayscale and color images, image type is automatically inferred.
During preprocessing raw image files are transformed into numpy ndarrays and saved in the hdf5 format.
    Images should have the same size.
    If they have different sizes they can be converted to the same size which should be set in the feature preprocessing
    parameters.</p>
<ul>
<li><code>in_memory</code> (default: <code>true</code>): defines whether image dataset will reside in memory during the training process or will be dynamically fetched from disk (useful for large datasets). In the latter case a training batch of input images will be fetched from disk each training iteration.</li>
    <li><code>resize_method</code> (default: <code>crop_or_pad</code>): available options: <code>crop_or_pad</code> -
        crops larger images to the desired size or pads smalled images using edge padding; <code>interpolate</code> -
        uses interpolation.
    </li>
<li><code>height</code> (default: null): image height in pixels, must be set if resizing is required</li>
<li><code>width</code> (default: null): image width in pixels, must be set if resizing is required</li>
</ul>
<h4 id="image-input-features-and-encoders">Image Input Features and Encoders<a class="headerlink" href="#image-input-features-and-encoders" title="Permanent link">&para;</a></h4>
                <p>Input image features are transformed into a float valued tensors of size <code>N x H x W x C</code>
                    (where <code>N</code> is the size of the dataset and <code>H x W</code> is a specific resizing of
                    the image that can be set, and <code>C</code> is the number of channels) and added to HDF5 with a
                    key that reflects the name of column in the CSV.
The column name is added to the JSON file, with an associated dictionary containing preprocessing information about the sizes of the resizing.</p>
                <p>Currently there are two encoders supported for images: Convolutional Stack Encoder and ResNet encoder
                    which can be set by setting <code>encoder</code> parameter to <code>stacked_cnn</code> or <code>resnet</code>
                    in the input feature dictionary in the model definition (<code>stacked_cnn</code> is the default
                    one).</p>
                <h5 id="convolutional-stack-encoder">Convolutional Stack Encoder<a class="headerlink"
                                                                                   href="#convolutional-stack-encoder"
                                                                                   title="Permanent link">&para;</a>
                </h5>
                <p>Convolutional Stack Encoder takes the following optional parameters:</p>
                <ul>
                    <li><code>conv_layers</code> (default <code>null</code>): it is a list of dictionaries containing
                        the parameters of all the convolutional layers. The length of the list determines the number of
                        stacked convolutional layers and the content of each dictionary determines the parameters for a
                        specific layer. The available parameters for each layer are: <code>filter_size</code>, <code>num_filters</code>,
                        <code>pool_size</code>, <code>norm</code>, <code>activation</code> and <code>regularize</code>.
                        If any of those values is missing from the dictionary, the default one specified as a parameter
                        of the encoder will be used instead. If both <code>conv_layers</code> and
                        <code>num_conv_layers</code> are <code>null</code>, a default list will be assigned to <code>conv_layers</code>
                        with the value <code>[{filter_size: 7, pool_size: 3, regularize: false}, {filter_size: 7,
                            pool_size: 3, regularize: false}, {filter_size: 3, pool_size: null, regularize: false},
                            {filter_size: 3, pool_size: null, regularize: false}, {filter_size: 3, pool_size: null,
                            regularize: true}, {filter_size: 3, pool_size: 3, regularize: true}]</code>.
                    </li>
                    <li><code>num_conv_layers</code> (default <code>null</code>): if <code>conv_layers</code> is <code>null</code>,
                        this is the number of stacked convolutional layers.
                    </li>
                    <li><code>filter_size</code> (default <code>3</code>): if a <code>filter_size</code> is not already
                        specified in <code>conv_layers</code> this is the default <code>filter_size</code> that will be
                        used for each layer. It indicates how wide is the 1d convolutional filter.
                    </li>
                    <li><code>num_filters</code> (default <code>256</code>): if a <code>num_filters</code> is not
                        already specified in <code>conv_layers</code> this is the default <code>num_filters</code> that
                        will be used for each layer. It indicates the number of filters, and by consequence the output
                        channels of the 2d convolution.
                    </li>
                    <li><code>pool_stride</code> (default <code>1</code>): if a <code>pool_stride</code> is not already
                        specified in <code>conv_layers</code> this is the default <code>pool_stride</code> that will be
                        used for each layer.
                    </li>
                    <li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the
                        parameters of all the fully connected layers. The length of the list determines the number of
                        stacked fully connected layers and the content of each dictionary determines the parameters for
                        a specific layer. The available parameters for each layer are: <code>fc_size</code>,
                        <code>norm</code>, <code>activation</code> and <code>regularize</code>. If any of those values
                        is missing from the dictionary, the default one specified as a parameter of the encoder will be
                        used instead. If both <code>fc_layers</code> and <code>num_fc_layers</code> are
                        <code>null</code>, a default list will be assigned to <code>fc_layers</code> with the value
                        <code>[{fc_size: 512}, {fc_size: 256}]</code>. (only applies if <code>reduce_output</code> is
                        not <code>null</code>).
                    </li>
                    <li><code>num_fc_layers</code> (default <code>1</code>): This is the number of stacked fully
                        connected layers.
                    </li>
                    <li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already
                        specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used
                        for each layer. It indicates the size of the output of a fully connected layer.
                    </li>
                    <li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified
                        in <code>fc_layers</code> or <code>conv_layers</code> this is the default <code>norm</code> that
                        will be used for each layer. It indicates the norm of the output and it can be <code>null</code>,
                        <code>batch</code> or <code>layer</code>.
                    </li>
                    <li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not
                        already specified in <code>fc_layers</code> or <code>conv_layers</code> this is the default
                        <code>activation</code> that will be used for each layer. It indicates the activation function
                        applied to the output.
                    </li>
                    <li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer
                        after each layer.
                    </li>
                    <li><code>initializer</code> (default <code>null</code>): the initializer to use. If
                        <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code>
                        in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>,
                        <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>,
                        <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>,
                        <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>,
                        <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a
                        dictionary with a key <code>type</code> that identifies the type of initializer and other keys
                        for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters
                        of each initializer, please refer to <a
                                href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's
                            documentation</a>.
                    </li>
                    <li><code>regularize</code> (default <code>true</code>): if <code>true</code> the weights of the
                        layers are added to the set of weights that get regularized by a regularization loss (if the
                        <code>regularization_lambda</code> in <code>training</code> is greater than 0).
                    </li>
                </ul>
                <h5 id="resnet-encoder">ResNet Encoder<a class="headerlink" href="#resnet-encoder"
                                                         title="Permanent link">&para;</a></h5>
                <p>ResNet Encoder takes the following optional parameters:</p>
                <ul>
                    <li><code>resnet_size</code> (default <code>50</code>): A single integer for the size of the ResNet
                        model.
                    </li>
                    <li><code>num_filters</code> (default <code>16</code>): It indicates the number of filters, and by
                        consequence the output channels of the 2d convolution.
                    </li>
                    <li><code>kernel_size</code> (default <code>3</code>): The kernel size to use for convolution.</li>
                    <li><code>conv_stride</code> (default <code>1</code>): Stride size for the initial convolutional
                        layer.
                    </li>
                    <li><code>first_pool_size</code> (default <code>null</code>): Pool size to be used for the first
                        pooling layer. If none, the first pooling layer is skipped.
                    </li>
                    <li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the
                        parameters of all the fully connected layers. The length of the list determines the number of
                        stacked fully connected layers and the content of each dictionary determines the parameters for
                        a specific layer. The available parameters for each layer are: <code>fc_size</code>,
                        <code>norm</code>, <code>activation</code> and <code>regularize</code>. If any of those values
                        is missing from the dictionary, the default one specified as a parameter of the encoder will be
                        used instead. If both <code>fc_layers</code> and <code>num_fc_layers</code> are
                        <code>null</code>, a default list will be assigned to <code>fc_layers</code> with the value
                        <code>[{fc_size: 512}, {fc_size: 256}]</code>. (only applies if <code>reduce_output</code> is
                        not <code>null</code>).
                    </li>
                    <li><code>num_fc_layers</code> (default <code>1</code>): This is the number of stacked fully
                        connected layers.
                    </li>
                    <li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already
                        specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used
                        for each layer. It indicates the size of the output of a fully connected layer.
                    </li>
                    <li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified
                        in <code>fc_layers</code> or <code>conv_layers</code> this is the default <code>norm</code> that
                        will be used for each layer. It indicates the norm of the output and it can be <code>null</code>,
                        <code>batch</code> or <code>layer</code>.
                    </li>
                    <li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not
                        already specified in <code>fc_layers</code> or <code>conv_layers</code> this is the default
                        <code>activation</code> that will be used for each layer. It indicates the activation function
                        applied to the output.
                    </li>
                    <li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer
                        after each layer.
                    </li>
                    <li><code>initializer</code> (default <code>null</code>): the initializer to use. If
                        <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code>
                        in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>,
                        <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>,
                        <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>,
                        <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>,
                        <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a
                        dictionary with a key <code>type</code> that identifies the type of initializer and other keys
                        for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters
                        of each initializer, please refer to <a
                                href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's
                            documentation</a>.
                    </li>
                    <li><code>regularize</code> (default <code>true</code>): if <code>true</code> the weights of the
                        layers are added to the set of weights that get regularized by a regularization loss (if the
                        <code>regularization_lambda</code> in <code>training</code> is greater than 0).
                    </li>
                </ul>
<h4 id="image-output-features-and-decoders">Image Output Features and Decoders<a class="headerlink" href="#image-output-features-and-decoders" title="Permanent link">&para;</a></h4>
<p>There are no image decoders at the moment (WIP), so image cannot be used as output features.</p>
<h4 id="image-features-measures">Image Features Measures<a class="headerlink" href="#image-features-measures" title="Permanent link">&para;</a></h4>
<p>As no image decoders are available at the moment, there are also no image measures.</p>
<h3 id="combiners">Combiners<a class="headerlink" href="#combiners" title="Permanent link">&para;</a></h3>
<p>Combiners are the part of the model that take the outputs of the encoders of all input features and combine them before providing the combined representation to the different output decoders.
If you don't specify a combiner, the <code>concat</code> combiner will be used.</p>
<h4 id="concat-combiner">Concat Combiner<a class="headerlink" href="#concat-combiner" title="Permanent link">&para;</a></h4>
<p>The concat combiner assumes all outputs from encoders are tensors of size <code>b x h</code> where <code>b</code> is the batch size and <code>h</code> is the hidden dimension, which can be different for each input.
It concatenates along the <code>h</code> dimension, and then (optionally) passes the concatenated tensor through a stack of fully connected layers.
It returns the final <code>b x h'</code> tensor where <code>h'</code> is the size of the last fully connected layer or the sum of the sizes of the <code>h</code> of all inputs in the case there are no fully connected layers.
If there's only one input feature and no fully connected layers are specified, the output of the input feature is just passed through as output.</p>
<div class="codehilite"><pre><span></span>+-----------+
|Input      |
|Feature 1  +-+
+-----------+ |            +---------+
+-----------+ | +------+   |Fully    |
|...        +---&gt;Concat+---&gt;Connected+-&gt;
+-----------+ | +------+   |Layers   |
+-----------+ |            +---------+
|Input      +-+
|Feature N  |
+-----------+
</pre></div>


<p>These are the available parameters of a concat combiner</p>
<ul>
<li><code>fc_layers</code> (default <code>null</code>): it is a list of dictionaries containing the parameters of all the fully connected layers. The length of the list determines the number of stacked fully connected layers and the content of each dictionary determines the parameters for a specific layer. The available parameters for each layer are: <code>fc_size</code>, <code>norm</code>, <code>activation</code>, <code>dropout</code>, <code>initializer</code> and <code>regularize</code>. If any of those values is missing from the dictionary, the default one specified as a parameter of the decoder will be used instead.</li>
<li><code>num_fc_layers</code> (default 0): this is the number of stacked fully connected layers that the input to the feature passes through. Their output is projected in the feature's output space.</li>
<li><code>fc_size</code> (default <code>256</code>): if a <code>fc_size</code> is not already specified in <code>fc_layers</code> this is the default <code>fc_size</code> that will be used for each layer. It indicates the size of the output of a fully connected layer.</li>
<li><code>activation</code> (default <code>relu</code>): if an <code>activation</code> is not already specified in <code>fc_layers</code> this is the default <code>activation</code> that will be used for each layer. It indicates the activation function applied to the output.</li>
<li><code>norm</code> (default <code>null</code>): if a <code>norm</code> is not already specified in <code>fc_layers</code> this is the default <code>norm</code> that will be used for each layer. It indicates the norm of the output and it can be <code>null</code>, <code>batch</code> or <code>layer</code>.</li>
<li><code>dropout</code> (default <code>false</code>): determines if there should be a dropout layer after each layer.</li>
<li><code>initializer</code> (default <code>null</code>): the initializer to use. If <code>null</code>, the default initialized of each variable is used (<code>glorot_uniform</code> in most cases). Options are: <code>constant</code>, <code>identity</code>, <code>zeros</code>, <code>ones</code>, <code>orthogonal</code>, <code>normal</code>, <code>uniform</code>, <code>truncated_normal</code>, <code>variance_scaling</code>, <code>glorot_normal</code>, <code>glorot_uniform</code>, <code>xavier_normal</code>, <code>xavier_uniform</code>, <code>he_normal</code>, <code>he_uniform</code>, <code>lecun_normal</code>, <code>lecun_uniform</code>. Alternatively it is possible to specify a dictionary with a key <code>type</code> that identifies the type of initializer and other keys for its parameters, e.g. <code>{type: normal, mean: 0, stddev: 0}</code>. To know the parameters of each initializer, please refer to <a href="https://www.tensorflow.org/api_docs/python/tf/keras/initializers">TensorFlow's documentation</a>.</li>
<li><code>regularize</code> (default <code>true</code>): if <code>true</code> the weights of the layers are added to the set of weights that get regularized by a regularization loss (if the <code>regularization_lambda</code> in <code>training</code> is greater than 0).</li>
</ul>
<p>Example concat combiner in the model definition:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">type</span><span
                        class="p p-Indicator">:</span> <span class="l l-Scalar l-Scalar-Plain">concat</span>
<span class="l l-Scalar l-Scalar-Plain">fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">num_fc_layers</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">0</span>
<span class="l l-Scalar l-Scalar-Plain">fc_size</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">256</span>
<span class="l l-Scalar l-Scalar-Plain">activation</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">relu</span>
<span class="l l-Scalar l-Scalar-Plain">norm</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">dropout</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">false</span>
<span class="l l-Scalar l-Scalar-Plain">initializer</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">regularize</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">true</span>
</pre></div>


<h4 id="sequence-concat-combiner">Sequence Concat Combiner<a class="headerlink" href="#sequence-concat-combiner" title="Permanent link">&para;</a></h4>
<p>The sequence concat combiner assumes at least one output from encoders is a tensors of size <code>b x s x h</code> where <code>b</code> is the batch size, <code>s</code> is the length of the sequence and <code>h</code> is the hidden dimension.
The sequence / text / sequential input can be specified with the <code>main_sequence_feature</code> parameter that should have the name of the sequential feature as value.
If no <code>main_sequence_feature</code> is specified, the combiner will look through all the features in the order they are defined in the model definition and will look for a feature with a rank 3 tensor output (sequence, text or time series).
If it cannot find one it will raise an exception, otherwise the output of that feature will be used for concatenating the other features along the sequence <code>s</code> dimension.</p>
<p>If there are other input features with a rank 3 output tensor, the combiner will concatenate them alongside the <code>s</code> dimension, which means that all of them must have identical <code>s</code> dimension, otherwise an error will be thrown.
    Specifically, as the placeholders of the sequential features are of dimension <code>[None, None]</code> in order to
    make the <code>BucketedBatcher</code> trim longer sequences to their actual length, the check if the sequences are
    of the same length cannot be performed at model building time, and a dimension mismatch error will be returned
    during training when a datapoint with two sequential features of different lengths are provided.</p>
<p>Other features that have a <code>b x h</code> rank 2 tensor output will be replicated <code>s</code> times and concatenated to the <code>s</code> dimension.
The final output is a <code>b x s x h'</code> tensor where <code>h'</code> is the size of the concatenation of the <code>h</code> dimensions of all input features.</p>
<div class="codehilite"><pre><span></span>+-----------+
|Input      |
|Feature 1  +-+
+-----------+ |            +---------+
+-----------+ | +------+   |Fully    |
|...        +---&gt;Concat+---&gt;Connected+-&gt;
+-----------+ | +------+   |Layers   |
+-----------+ |            +---------+
|Input      +-+
|Feature N  |
+-----------+
</pre></div>


<p>These are the available parameters of a sequence concat combiner</p>
<ul>
<li><code>main_sequence_feature</code> (default <code>null</code>): name fo the sequence / text/ time series feature to concatenate the outputs of the other features to. If no <code>main_sequence_feature</code> is specified, the combiner will look through all the features in the order they are defined in the model definition and will look for a feature with a rank 3 tensor output (sequence, text or time series). If it cannot find one it will raise an exception, otherwise the output of that feature will be used for concatenating the other features along the sequence <code>s</code> dimension. If there are other input features with a rank 3 output tensor, the combiner will concatenate them alongside the <code>s</code> dimension, which means that all of them must have identical <code>s</code> dimension, otherwise an error will be thrown.</li>
</ul>
<p>Example sequence concat combiner in the model definition:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">type</span><span
                        class="p p-Indicator">:</span> <span class="l l-Scalar l-Scalar-Plain">sequence_concat</span>
<span class="l l-Scalar l-Scalar-Plain">main_sequence_feature</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
</pre></div>


<h4 id="sequence-combiner">Sequence Combiner<a class="headerlink" href="#sequence-combiner" title="Permanent link">&para;</a></h4>
<p>The sequence combiner stacks a sequence concat combiner with a sequence encoder one on top of each other.
All the considerations about inputs tensor ranks describer for the <a href="#sequence-concat-combiner">sequence concat combiner</a> apply also in this case, but the main difference is that this combiner uses the <code>b x s x h'</code> output of the sequence concat combiner, where <code>b</code> is the batch size, <code>s</code> is the sequence length and <code>h'</code> is the sum of the hidden dimensions of all input features, as input fo any of the sequence encoders described in the <a href="#sequence-inpit-features-and-encoders">sequence features encoders section</a>.
Refer to that section for more detailed information about the sequence encoders and their parameters.
Also all the considerations on the shape of the outputs done for the sequence encoders apply in this case too.</p>
<div class="codehilite"><pre><span></span>Sequence
Feature
Output

+---------+
|emb seq 1|
+---------+
|...      +--+
+---------+  |  +-----------------+
|emb seq n|  |  |emb seq 1|emb oth|   +--------+
+---------+  |  +-----------------+   |Sequence|
             +--&gt;...      |...    +--&gt;+Encoder +-&gt;
Other        |  +-----------------+   |        |
Feature      |  |emb seq n|emb oth|   +--------+
Output       |  +-----------------+
             |
+-------+    |
|emb oth+----+
+-------+
</pre></div>


<p>Example sequence concat combiner in the model definition:</p>
                <div class="codehilite"><pre><span></span><span class="l l-Scalar l-Scalar-Plain">type</span><span
                        class="p p-Indicator">:</span> <span class="l l-Scalar l-Scalar-Plain">sequence</span>
<span class="l l-Scalar l-Scalar-Plain">main_sequence_feature</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">null</span>
<span class="l l-Scalar l-Scalar-Plain">encoder</span><span class="p p-Indicator">:</span> <span
                            class="l l-Scalar l-Scalar-Plain">parallel_cnn</span>
<span class="nn">...</span> <span class="l l-Scalar l-Scalar-Plain">encoder parameters ...</span>
</pre></div>


<h2 id="distributed-training">Distributed Training<a class="headerlink" href="#distributed-training" title="Permanent link">&para;</a></h2>
<p>You can distribute the training and prediction of your models using <a href="https://github.com/uber/horovod">Horovod</a>, which allows to train on a single machine with multiple GPUs as well as on multiple machines with multiple GPUs.</p>
<p>In order to use distributed training you have to install Horovod as detailed in <a href="https://github.com/uber/horovod#install">Horovod's installation instructions</a> (which include installing <a href="https://www.open-mpi.org">OpenMPI</a> or other <a href="https://en.wikipedia.org/wiki/Message_Passing_Interface">MPI</a> implementations) and then install the two packages:</p>
<div class="codehilite"><pre><span></span>pip install horovod mpi4py
</pre></div>


<p>Horovod works by, in practice, increasing the batch size and distributing a part of each batch to a different node and collecting the gradients from all the nodes in a smart and scalable way.
It also adjusts the learning rate to counter balance the increase in the batch size.
The advantage is that training speed scales almost linearly with the number of nodes.</p>
<p><code>experiment</code>, <code>train</code> and <code>predict</code> commands accept a <code>--horovod</code> argument that instructs the model building, training and prediction phases to be conducted using Horovod in a distributed way.
An MPI command specifying which machines and / or GPUs to use, together with a few more parameters, must be provided before the call to Ludwig's command.
For instance, in order to train a Ludwig model on a local machine with four GPUs one you can run:</p>
<div class="codehilite"><pre><span></span>mpirun -np 4 \
    -H localhost:4 \
    -bind-to none -map-by slot \
    -x NCCL_DEBUG=INFO -x LD_LIBRARY_PATH -x PATH \
    -mca pml ob1 -mca btl ^openib \
    ludwig train --horovod ...other Ludwig parameters...
</pre></div>


<p>While for training on four remote machines with four GPUs each you can run:</p>
<div class="codehilite"><pre><span></span>mpirun -np 16 \
    -H server1:4,server2:4,server3:4,server4:4 \
    -bind-to none -map-by slot \
    -x NCCL_DEBUG=INFO -x LD_LIBRARY_PATH -x PATH \
    -mca pml ob1 -mca btl ^openib \
    ludwig train --horovod ...other Ludwig parameters...
</pre></div>


<p>The same applies to <code>experiment</code> and <code>predict</code>.</p>
<p>More details on the installation of MPI and how to run Horovod can be found in <a href="https://github.com/uber/horovod">Horovod's documentation</a>.</p>
<h2 id="programatic-api">Programatic API<a class="headerlink" href="#programatic-api" title="Permanent link">&para;</a></h2>
<p>Ludwig functionalities can also be accessed through a programmatic API.
The API consists of one <code>LudwigModel</code> class that can be initialized with a model definition dictionary and then can be trained with data coming in the form of a dataframe or a CSV file.
Pretrained models can be loaded and can be used to obtain predictions on new data, again either in dataframe or CSV format.</p>
<p>A detailed documentation of all the functions available in <code>LudwigModel</code> is provided in the <a href="../api/">API documentation</a>.</p>
<h3 id="training-a-model">Training a Model<a class="headerlink" href="#training-a-model" title="Permanent link">&para;</a></h3>
<p>To train a model one has first to initialize it using the initializer <code>LudwigModel()</code> and a model definition dictionary, and then calling the <code>train()</code> function using either a dataframe or a CSV file.</p>
<div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">ludwig</span> <span class="kn">import</span> <span class="n">LudwigModel</span>

<span class="n">model_definition</span> <span class="o">=</span> <span class="p">{</span><span class="o">...</span><span class="p">}</span>
<span class="n">model</span> <span class="o">=</span> <span class="n">LudwigModel</span><span class="p">(</span><span class="n">model_definition</span><span class="p">)</span>
<span class="n">train_stats</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="n">data_csv</span><span class="o">=</span><span class="n">csv_file_path</span><span class="p">)</span>
<span class="c1"># or</span>
<span class="n">train_stats</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="n">data_df</span><span class="o">=</span><span class="n">dataframe</span><span class="p">)</span>
</pre></div>


<p><code>model_definition</code> is a dictionary that has the same key-value structure of a model definition YAML file, as it's technically equivalent as parsing the YAML file into a Python dictionary.
<code>train_stats</code> will be a dictionary containing statistics about the training.
The contents are exactly the same of the <code>training_statistics.json</code> file produced by the <code>experiment</code> and <code>train</code> commands.</p>
<h3 id="loading-a-pre-trained-model">Loading a Pre-trained Model<a class="headerlink" href="#loading-a-pre-trained-model" title="Permanent link">&para;</a></h3>
<p>In order to load a pre-trained Ludwig model you have to call the static function <code>load()</code> of the <code>LudwigModel</code> class providing the path containing the model.</p>
<div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">ludwig</span> <span class="kn">import</span> <span class="n">LudwigModel</span>

<span class="n">model</span> <span class="o">=</span> <span class="n">LudwigModel</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">model_path</span><span class="p">)</span>
</pre></div>


<h3 id="predicting">Predicting<a class="headerlink" href="#predicting" title="Permanent link">&para;</a></h3>
<p>Either a newly trained model or a pre-trained loaded model can be used for predicting on new data using the <code>predict()</code> function of the model object.
The CSV / dataframe has to contain columns with the same names of all the input features of the model.</p>
<div class="codehilite"><pre><span></span><span class="n">predictions</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">dataset_csv</span><span class="o">=</span><span class="n">csv_file_path</span><span class="p">)</span>
<span class="c1">#or</span>
<span class="n">predictions</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">dataset_df</span><span class="o">=</span><span class="n">dataframe</span><span class="p">)</span>
</pre></div>


<p><code>predictions</code> will be a dataframe containing the prediction and confidence score / probability of all output features.</p>
<p>If you want to compute also measures on the quality of the predictions you can run:</p>
<div class="codehilite"><pre><span></span><span class="n">predictions</span><span class="p">,</span> <span class="n">test_stats</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">test</span><span class="p">(</span><span class="n">dataset_csv</span><span class="o">=</span><span class="n">csv_file_path</span><span class="p">)</span>
<span class="c1">#or</span>
<span class="n">predictions</span><span class="p">,</span> <span class="n">test_stats</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">test</span><span class="p">(</span><span class="n">dataset_df</span><span class="o">=</span><span class="n">dataframe</span><span class="p">)</span>
</pre></div>


<p>In this case the CSV / dataframe should also contain columns with the same names of all the output features, as their content is going to be used as ground truth to compare the predictions against and compute the measures and <code>test_stats</code> will be a dictionary containing several measures of quality depending on the type of each output feature (e.g. <code>category</code> features will have an accuracy measure and a confusion matrix, among other measures, associated to them, while <code>numerical</code> features will have measures like mean squared loss and R2 among others).</p>
<h2 id="visualizations">Visualizations<a class="headerlink" href="#visualizations" title="Permanent link">&para;</a></h2>
<p>Several visualization can be obtained from the result files from both <code>train</code>, <code>predict</code> and <code>experiment</code> by using the <code>visualize</code> command.
The command has several parameters, but not all the visualizations use all of them.
Let's first present the parameters of the general script, and then, for each available visualization, we will discuss about the specific parameters needed and what visualization they produce.</p>
<div class="codehilite"><pre><span></span>usage: ludwig visualize [options]

This script analyzes results and shows some nice plots.

optional arguments:
  -h, --help            show this help message and exit
  -d DATA_CSV, --data_csv DATA_CSV
                        raw data file
  -g GROUND_TRUTH, --ground_truth GROUND_TRUTH
                        ground truth file
  -gm GROUND_TRUTH_METADATA, --ground_truth_metadata GROUND_TRUTH_METADATA
                        input metadata JSON file
  -v {learning_curves,compare_performance,compare_classifiers_performance_from_prob,compare_classifiers_performance_from_pred,compare_classifiers_performance_subset,compare_classifiers_performance_changing_k,compare_classifiers_predictions,compare_classifiers_predictions_distribution,confidence_thresholding,confidence_thresholding_data_vs_acc,confidence_thresholding_data_vs_acc_subset,confidence_thresholding_data_vs_acc_subset_per_class,confidence_thresholding_2thresholds_2d,confidence_thresholding_2thresholds_3d,binary_threshold_vs_metric,roc_curves,roc_curves_from_prediction_statistics,calibration_1_vs_all,calibration_multiclass,confusion_matrix,compare_classifiers_multiclass_multimetric,frequency_vs_f1}, --visualization {learning_curves,compare_performance,compare_classifiers_performance_from_prob,compare_classifiers_performance_from_pred,compare_classifiers_performance_subset,compare_classifiers_performance_changing_k,compare_classifiers_predictions,compare_classifiers_predictions_distribution,confidence_thresholding,confidence_thresholding_data_vs_acc,confidence_thresholding_data_vs_acc_subset,confidence_thresholding_data_vs_acc_subset_per_class,confidence_thresholding_2thresholds_2d,confidence_thresholding_2thresholds_3d,binary_threshold_vs_metric,roc_curves,roc_curves_from_prediction_statistics,calibration_1_vs_all,calibration_multiclass,confusion_matrix,compare_classifiers_multiclass_multimetric,frequency_vs_f1}
                        type of visualization
  -f FIELD, --field FIELD
                        field containing ground truth
  -tf THRESHOLD_FIELDS [THRESHOLD_FIELDS ...], --threshold_fields THRESHOLD_FIELDS [THRESHOLD_FIELDS ...]
                        fields for 2d threshold
  -pred PREDICTIONS [PREDICTIONS ...], --predictions PREDICTIONS [PREDICTIONS ...]
                        predictions files
  -prob PROBABILITIES [PROBABILITIES ...], --probabilities PROBABILITIES [PROBABILITIES ...]
                        probabilities files
  -ts TRAINING_STATISTICS [TRAINING_STATISTICS ...], --training_statistics TRAINING_STATISTICS [TRAINING_STATISTICS ...]
                        training stats files
  -ps PREDICTION_STATISTICS [PREDICTION_STATISTICS ...], --prediction_statistics PREDICTION_STATISTICS [PREDICTION_STATISTICS ...]
                        test stats files
  -mn MODEL_NAMES [MODEL_NAMES ...], --model_names MODEL_NAMES [MODEL_NAMES ...]
                        names of the models to use as labels
  -tn TOP_N_CLASSES [TOP_N_CLASSES ...], --top_n_classes TOP_N_CLASSES [TOP_N_CLASSES ...]
                        number of classes to plot
  -k TOP_K, --top_k TOP_K
                        number of elements in the ranklist to consider
  -ll LABELS_LIMIT, --labels_limit LABELS_LIMIT
                        maximum numbers of labels. If labels in dataset are
                        higher than this number, &quot;rare&quot; label
  -ss {ground_truth,predictions}, --subset {ground_truth,predictions}
                        type of subset filtering
  -n, --normalize       normalize rows in confusion matrix
  -m METRICS [METRICS ...], --metrics METRICS [METRICS ...]
                        metrics to display in threshold_vs_metric
  -pl POSITIVE_LABEL, --positive_label POSITIVE_LABEL
                        label of the positive class for the roc curve
  -l {critical,error,warning,info,debug,notset}, --logging_level {critical,error,warning,info,debug,notset}
                        the level of logging to use
</pre></div>


<p>Some additional information on the parameters:</p>
<ul>
<li>The list parameters are considered to be aligned, meaning <code>predictions</code>, <code>probabilities</code>, <code>training_statistics</code>, <code>prediction_statistics</code> and <code>model_names</code> are indexed altogether, for instance the name of the model producing the second predictions in the list will be the second in the model names.</li>
<li><code>data_csv</code> is intended to be the data the model(s) were trained on.</li>
<li><code>ground_truth</code> and <code>ground_truth_metadata</code> are respectively the <code>HDF5</code> and <code>JSON</code> file obtained during training preprocessing. If you plan to use the visualizations then be sure not to use the <code>skip_save_preprocessing</code> when training. Those files are needed because they contain the split performed at preprocessing time, so it is easy to extract the test set from them.</li>
<li><code>field</code> is the output feature to use for creating the visualization.</li>
</ul>
<p>Other parameters will be detailed for each visualization as different ones use them differently.</p>
<h3 id="learning-curves">Learning Curves<a class="headerlink" href="#learning-curves" title="Permanent link">&para;</a></h3>
<h4 id="learning_curves">learning_curves<a class="headerlink" href="#learning_curves" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>training_statistics</code> and <code>model_names</code> parameters.
For each model (in the aligned lists of <code>training_statistics</code> and <code>model_names</code>) and for each output feature and measure of the model, it produces a line plot showing how that measure changed over the course of the epochs of training on the training and validation sets.</p>
<p><img alt="Learning Curves Loss" src="../images/learning_curves_loss.png" title="Learning Curves Loss" /></p>
<p><img alt="Learning Curves Accuracy" src="../images/learning_curves_accuracy.png" title="Learning Curves Accuracy" /></p>
<h3 id="confusion-matrix">Confusion Matrix<a class="headerlink" href="#confusion-matrix" title="Permanent link">&para;</a></h3>
<h4 id="confusion_matrix">confusion_matrix<a class="headerlink" href="#confusion_matrix" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>top_n_classes</code>, <code>normalize</code>, <code>ground_truth_metadata</code>, <code>prediction_statistics</code> and <code>model_names</code> parameters.
For each model (in the aligned lists of <code>prediction_statistics</code> and <code>model_names</code>) it produces a heatmap of the confusion matrix in the predictions for each field that has a confusion matrix in <code>prediction_statistics</code>.
The value of <code>top_n_classes</code> limits the heatmap to the <code>n</code> most frequent classes.</p>
<p><img alt="Confusion Matrix" src="../images/confusion_matrix.png" title="Confusion Matrix" /></p>
<p>The second plot produced, is a barplot showing the entropy of each class, ranked from most entropic to least entropic.</p>
<p><img alt="Confusion Matrix Entropy" src="../images/confusion_matrix_entropy.png" title="Confusion Matrix Entropy" /></p>
<h3 id="compare-performance">Compare Performance<a class="headerlink" href="#compare-performance" title="Permanent link">&para;</a></h3>
<h4 id="compare_performance">compare_performance<a class="headerlink" href="#compare_performance" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>field</code>, <code>prediction_statistics</code> and <code>model_names</code> parameters.
For each model (in the aligned lists of <code>prediction_statistics</code> and <code>model_names</code>) it produces bars in a bar plot, one for each overall metric available in the <code>prediction_statistics</code> file for the specified <code>field</code>.</p>
<p><img alt="Compare Classifiers Performance" src="../images/compare_performance.png" title="Compare Classifiers Performance" /></p>
<h4 id="compare_classifiers_performance_from_prob">compare_classifiers_performance_from_prob<a class="headerlink" href="#compare_classifiers_performance_from_prob" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>ground_truth</code>, <code>field</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
For each model (in the aligned lists of <code>probabilities</code> and <code>model_names</code>) it produces bars in a bar plot, one for each overall metric computed on the fly from the probabilities of predictions for the specified <code>field</code>.</p>
<p><img alt="Compare Classifiers Performance from Probabilities" src="../images/compare_classifiers_performance_from_prob.png" title="Compare Classifiers Performance from probabilities" /></p>
<h4 id="compare_classifiers_performance_from_pred">compare_classifiers_performance_from_pred<a class="headerlink" href="#compare_classifiers_performance_from_pred" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>ground_truth</code>, <code>ground_truth_metadata</code>, <code>field</code>, <code>predictions</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
For each model (in the aligned lists of <code>predictions</code> and <code>model_names</code>) it produces bars in a bar plot, one for each overall metric computed on the fly from the predictions for the specified <code>field</code>.</p>
<p><img alt="Compare Classifiers Performance from Predictions" src="../images/compare_classifiers_performance_from_pred.png" title="Compare Classifiers Performance from Predictions" /></p>
<h4 id="compare_classifiers_performance_subset">compare_classifiers_performance_subset<a class="headerlink" href="#compare_classifiers_performance_subset" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>top_n_classes</code>, <code>subset</code>, <code>ground_truth</code>, <code>ground_truth_metadata</code>, <code>field</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
For each model (in the aligned lists of <code>predictions</code> and <code>model_names</code>) it produces bars in a bar plot, one for each overall metric computed on the fly from the probabilities predictions for the specified <code>field</code>, considering only a subset of the full training set.
The way the subset is obtained is using the <code>top_n_classes</code> and <code>subset</code> parameters.</p>
<p>If the values of <code>subset</code> is <code>ground_truth</code>, then only datapoints where the ground truth class is within the top <code>n</code> most frequent ones will be considered as test set, and the percentage of datapoints that have been kept from the original set will be displayed.</p>
<p><img alt="Compare Classifiers Performance Subset Ground Truth" src="../images/compare_classifiers_performance_subset_gt.png" title="Compare Classifiers Performance Subset Ground Truth" /></p>
<p>If the values of <code>subset</code> is <code>predictions</code>, then only datapoints where the the model predicts a class that is within the top <code>n</code> most frequent ones will be considered as test set, and the percentage of datapoints that have been kept from the original set will be displayed for each model.</p>
<p><img alt="Compare Classifiers Performance Subset Ground Predictions" src="../images/compare_classifiers_performance_subset_pred.png" title="Compare Classifiers Performance Subset Ground Predictions" /></p>
<h4 id="compare_classifiers_performance_changing_k">compare_classifiers_performance_changing_k<a class="headerlink" href="#compare_classifiers_performance_changing_k" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>top_k</code>, <code>ground_truth_metadata</code>, <code>field</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
For each model (in the aligned lists of <code>probabilities</code> and <code>model_names</code>) it produces a line plot that shows the Hits@K measure (that counts a prediction as correct if the model produces it among the first <code>k</code>) while changing <code>k</code> from 1 to <code>top_k</code> for the specified <code>field</code>.</p>
<p><img alt="Compare Classifiers Performance Changing K" src="../images/compare_classifiers_performance_changing_k.png" title="Compare Classifiers Performance  Changing K" /></p>
<h4 id="compare_classifiers_multiclass_multimetric">compare_classifiers_multiclass_multimetric<a class="headerlink" href="#compare_classifiers_multiclass_multimetric" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>top_n_classes</code>, <code>ground_truth_metadata</code>, <code>field</code>, <code>prediction_statistics</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
For each model (in the aligned lists of <code>prediction_statistics</code> and <code>model_names</code>) it produces four plots that show the precision, recall and F1 of the model on several classes for the specified <code>field</code>.</p>
<p>The first one show the measures on the <code>n</code> most frequent classes.</p>
<p><img alt="Multiclass Multimetric top k" src="../images/compare_classifiers_multiclass_multimetric_topk.png" title="Multiclass Multimetric most frequent classes" /></p>
<p>The second one shows the measures on the <code>n</code> classes where the model performs the best.</p>
<p><img alt="Multiclass Multimetric best k" src="../images/compare_classifiers_multiclass_multimetric_bestk.png" title="Multiclass Multimetric best classes" /></p>
<p>The third one shows the measures on the <code>n</code> classes where the model performs the worst.</p>
<p><img alt="Multiclass Multimetric worst k" src="../images/compare_classifiers_multiclass_multimetric_worstk.png" title="Multiclass Multimetric worst classes" /></p>
<p>The fourth one shows the measures on all the classes, sorted by their frequency. This could become unreadable in case the number of classes is really high.</p>
<p><img alt="Multiclass Multimetric sorted" src="../images/compare_classifiers_multiclass_multimetric_sorted.png" title="Multiclass Multimetric sorted classes" /></p>
<h3 id="compare-classifier-predictions">Compare Classifier Predictions<a class="headerlink" href="#compare-classifier-predictions" title="Permanent link">&para;</a></h3>
<h4 id="compare_classifiers_predictions">compare_classifiers_predictions<a class="headerlink" href="#compare_classifiers_predictions" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>ground_truth</code>, <code>ground_truth_metadata</code>, <code>field</code>, <code>predictions</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category and there must be two and only two models (in the aligned lists of <code>predictions</code> and <code>model_names</code>).
This visualization produces a pie chart comparing the predictions of the two models for the specified <code>field</code>.</p>
<p><img alt="Compare Classifiers Predictions" src="../images/compare_classifiers_predictions.png" title="Compare Classifiers Predictions" /></p>
<h4 id="compare_classifiers_predictions_distribution">compare_classifiers_predictions_distribution<a class="headerlink" href="#compare_classifiers_predictions_distribution" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>ground_truth</code>, <code>ground_truth_metadata</code>, <code>field</code>, <code>predictions</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
This visualization produces a radar plot comparing the distributions of predictions of the models for the first 10 classes of the specified <code>field</code>.</p>
<p><img alt="Compare Classifiers Predictions Distribution" src="../images/compare_classifiers_predictions_distribution.png" title="Compare Classifiers Predictions Distribution" /></p>
<h3 id="confidence_thresholding">Confidence_Thresholding<a class="headerlink" href="#confidence_thresholding" title="Permanent link">&para;</a></h3>
<h4 id="confidence_thresholding_1">confidence_thresholding<a class="headerlink" href="#confidence_thresholding_1" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>ground_truth</code>, <code>field</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
For each model (in the aligned lists of <code>probabilities</code> and <code>model_names</code>) it produces a pair of lines indicating the accuracy of the model and the data coverage while increasing a threshold (x axis) on the probabilities of predictions for the specified <code>field</code>.</p>
<p><img alt="Confidence_Thresholding" src="../images/confidence_thresholding.png" title="Confidence_Thresholding" /></p>
<h4 id="confidence_thresholding_data_vs_acc">confidence_thresholding_data_vs_acc<a class="headerlink" href="#confidence_thresholding_data_vs_acc" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>ground_truth</code>, <code>field</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
For each model (in the aligned lists of <code>probabilities</code> and <code>model_names</code>) it produces a line indicating the accuracy of the model and the data coverage while increasing a threshold on the probabilities of predictions for the specified <code>field</code>.
The difference with <code>confidence_thresholding</code> is that it uses two axes instead of three, not visualizing the threshold and having coverage as x axis instead of the threshold.</p>
<p><img alt="Confidence_Thresholding Data vs Accuracy" src="../images/confidence_thresholding_data_vs_acc.png" title="Confidence_Thresholding Data vs Accuracy" /></p>
<h4 id="confidence_thresholding_data_vs_acc_subset">confidence_thresholding_data_vs_acc_subset<a class="headerlink" href="#confidence_thresholding_data_vs_acc_subset" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>top_n_classes</code>, <code>subset</code>, <code>ground_truth</code>, <code>field</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
For each model (in the aligned lists of <code>probabilities</code> and <code>model_names</code>) it produces a line indicating the accuracy of the model and the data coverage while increasing a threshold on the probabilities of predictions for the specified <code>field</code>, considering only a subset of the full training set.
The way the subset is obtained is using the <code>top_n_classes</code> and <code>subset</code> parameters..
The difference with <code>confidence_thresholding</code> is that it uses two axes instead of three, not visualizing the threshold and having coverage as x axis instead of the threshold.</p>
<p>If the values of <code>subset</code> is <code>ground_truth</code>, then only datapoints where the ground truth class is within the top <code>n</code> most frequent ones will be considered as test set, and the percentage of datapoints that have been kept from the original set will be displayed.
If the values of <code>subset</code> is <code>predictions</code>, then only datapoints where the the model predicts a class that is within the top <code>n</code> most frequent ones will be considered as test set, and the percentage of datapoints that have been kept from the original set will be displayed for each model.</p>
<p><img alt="Confidence_Thresholding Data vs Accuracy Subset" src="../images/confidence_thresholding_data_vs_acc_subset.png" title="Confidence_Thresholding Data vs Accuracy  Subset" /></p>
<h4 id="confidence_thresholding_data_vs_acc_subset_per_class">confidence_thresholding_data_vs_acc_subset_per_class<a class="headerlink" href="#confidence_thresholding_data_vs_acc_subset_per_class" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>top_n_classes</code>, <code>subset</code>, <code>ground_truth</code>, <code>ground_truth_metadata</code>, <code>field</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
For each model (in the aligned lists of <code>probabilities</code> and <code>model_names</code>) it produces a line indicating the accuracy of the model and the data coverage while increasing a threshold on the probabilities of predictions for the specified <code>field</code>, considering only a subset of the full training set.
The way the subset is obtained is using the <code>top_n_classes</code> and <code>subset</code> parameters..
The difference with <code>confidence_thresholding</code> is that it uses two axes instead of three, not visualizing the threshold and having coverage as x axis instead of the threshold.</p>
<p>If the values of <code>subset</code> is <code>ground_truth</code>, then only datapoints where the ground truth class is within the top <code>n</code> most frequent ones will be considered as test set, and the percentage of datapoints that have been kept from the original set will be displayed.
If the values of <code>subset</code> is <code>predictions</code>, then only datapoints where the the model predicts a class that is within the top <code>n</code> most frequent ones will be considered as test set, and the percentage of datapoints that have been kept from the original set will be displayed for each model.</p>
<p>The difference with <code>confidence_thresholding_data_vs_acc_subset</code> is that it produces one plot per class within the <code>top_n_classes</code>.</p>
<p><img alt="Confidence_Thresholding Data vs Accuracy Subset per class 1" src="../images/confidence_thresholding_data_vs_acc_subset_per_class_1.png" title="Confidence_Thresholding Data vs Accuracy Subset per class 1" /></p>
<p><img alt="Confidence_Thresholding Data vs Accuracy Subset per class 4" src="../images/confidence_thresholding_data_vs_acc_subset_per_class_4.png" title="Confidence_Thresholding Data vs Accuracy Subset per class 4" /></p>
<h4 id="confidence_thresholding_2thresholds_2d">confidence_thresholding_2thresholds_2d<a class="headerlink" href="#confidence_thresholding_2thresholds_2d" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>ground_truth</code>, <code>threshold_fields</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>threshold_fields</code> need to be exactly two, either category or binary.
<code>probabilities</code> need to be exactly two, aligned with <code>threshold_fields</code>.
<code>model_names</code> has to be exactly one.
Three plots are produced.</p>
<p>The first plot shows several semi transparent lines.
They summarize the 3d surfaces displayed by <code>confidence_thresholding_2thresholds_3d</code> that have thresholds on the confidence of the predictions of the two <code>threshold_fields</code> as x and y axes and either the data coverage percentage or the accuracy as z axis.
Each line represents a slice of the data coverage surface projected onto the accuracy surface.</p>
<p><img alt="Confidence_Thresholding two thresholds 2D Multiline" src="../images/confidence_thresholding_2thresholds_2d_multiline.png" title="Confidence_Thresholding two thresholds 2D Multiline" /></p>
<p>The second plot shows the max of all the lines displayed in the first plot.</p>
<p><img alt="Confidence_Thresholding two thresholds 2D Maxline" src="../images/confidence_thresholding_2thresholds_2d_maxline.png" title="Confidence_Thresholding two thresholds 2D Maxline" /></p>
<p>The third plot shows the max line and the values of the thresholds that obtained a specific data coverage vs accuracy pair of values.</p>
<p><img alt="Confidence_Thresholding two thresholds 2D Accuracy and Thresholds" src="../images/confidence_thresholding_2thresholds_2d_accthr.png" title="Confidence_Thresholding two thresholds 2D Accuracy and Thresholds" /></p>
<h4 id="confidence_thresholding_2thresholds_3d">confidence_thresholding_2thresholds_3d<a class="headerlink" href="#confidence_thresholding_2thresholds_3d" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>ground_truth</code>, <code>threshold_fields</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>threshold_fields</code> need to be exactly two, either category or binary.
<code>probabilities</code> need to be exactly two, aligned with <code>threshold_fields</code>.
<code>model_names</code> has to be exactly one.
The plot shows the 3d surfaces displayed by <code>confidence_thresholding_2thresholds_3d</code> that have thresholds on the confidence of the predictions of the two <code>threshold_fields</code> as x and y axes and either the data coverage percentage or the accuracy as z axis.</p>
<p><img alt="Confidence_Thresholding two thresholds 3D" src="../images/confidence_thresholding_2thresholds_3d.png" title="Confidence_Thresholding two thresholds 3D" /></p>
<h3 id="binary-threshold-vs-metric">Binary Threshold vs. Metric<a class="headerlink" href="#binary-threshold-vs-metric" title="Permanent link">&para;</a></h3>
<h4 id="binary_threshold_vs_metric">binary_threshold_vs_metric<a class="headerlink" href="#binary_threshold_vs_metric" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>positive_label</code>, <code>metrics</code>, <code>ground_truth</code>, <code>ground_truth_metadata</code>, <code>field</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>field</code> can be a category or binary feature.
For each metric specified in <code>metrics</code> (options are <code>f1</code>, <code>precision</code>, <code>recall</code>, <code>accuracy</code>), this visualization produces a line chart plotting a threshold on the confidence of the model against the metric for the specified <code>field</code>.
If <code>field</code> is a category feature, <code>positive_label</code> indicates which is the class to be considered positive class and all the others will be considered negative.
It needs to be an integer, to figure out the association between classes and integers check the <code>ground_truth_metadata</code> JSON file.</p>
<p><img alt="Binary_Threshold_vs_Metric" src="../images/binary_threshold_vs_metric.png" title="Binary_Threshold_vs_Metric" /></p>
<h3 id="roc-curves">ROC Curves<a class="headerlink" href="#roc-curves" title="Permanent link">&para;</a></h3>
<h4 id="roc_curves">roc_curves<a class="headerlink" href="#roc_curves" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>positive_label</code>, <code>ground_truth</code>, <code>ground_truth_metadata</code>, <code>field</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>field</code> can be a category or binary feature.
This visualization produces a line chart plotting the roc curves for the specified <code>field</code>.
If <code>field</code> is a category feature, <code>positive_label</code> indicates which is the class to be considered positive class and all the others will be considered negative.
It needs to be an integer, to figure out the association between classes and integers check the <code>ground_truth_metadata</code> JSON file.</p>
<p><img alt="ROC Curves" src="../images/roc_curves.png" title="ROC Curves" /></p>
<h4 id="roc_curves_from_test_stats">roc_curves_from_test_stats<a class="headerlink" href="#roc_curves_from_test_stats" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>field</code>, <code>prediction_statistics</code> and <code>model_names</code> parameters.
<code>field</code> needs to be binary feature.
This visualization produces a line chart plotting the roc curves for the specified <code>field</code>.</p>
<p><img alt="ROC Curves from Prediction Statistics" src="../images/roc_curves_from_prediction_statistics.png" title="ROC Curves from Prediction Statistics" /></p>
<h3 id="calibration-plot">Calibration Plot<a class="headerlink" href="#calibration-plot" title="Permanent link">&para;</a></h3>
<h4 id="calibration_1_vs_all">calibration_1_vs_all<a class="headerlink" href="#calibration_1_vs_all" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>top_k</code>, <code>ground_truth</code>, <code>field</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category or binary.
For each class or each of the <code>k</code> most frequent classes if <code>top_k</code> is specified, it produces two plots computed on the fly from the probabilities of predictions for the specified <code>field</code>.</p>
<p>The first plot is a calibration curve that shows the calibration of the predictions considering the current class to be the true one and all others to be a false one, drawing one line for each model (in the aligned lists of <code>probabilities</code> and <code>model_names</code>).</p>
<p><img alt="Calibration 1 vs All Curve" src="../images/calibration_1_vs_all_curve.png" title="Calibration 1 vs All Curve" /></p>
<p>The second plot shows the distributions of the predictions considering the current class to be the true one and all others to be a false one, drawing the distribution for each model (in the aligned lists of <code>probabilities</code> and <code>model_names</code>).</p>
<p><img alt="Calibration 1 vs All Counts" src="../images/calibration_1_vs_all_counts.png" title="Calibration 1 vs All Counts" /></p>
<h4 id="calibration_multiclass">calibration_multiclass<a class="headerlink" href="#calibration_multiclass" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>ground_truth</code>, <code>field</code>, <code>probabilities</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
For each class, produces two plots computed on the fly from the probabilities of predictions for the specified <code>field</code>.</p>
<p>The first plot is a calibration curve that shows the calibration of the predictions considering al classes, drawing one line for each model (in the aligned lists of <code>probabilities</code> and <code>model_names</code>).</p>
<p><img alt="Calibration Multiclass Curve" src="../images/calibration_multiclass_curve.png" title="Calibration Multiclass Curve" /></p>
<p>The second plot shows a bar plot of the brier score (that calculates how calibrated are the probabilities of the predictions of a model), drawing one bar for each model (in the aligned lists of <code>probabilities</code> and <code>model_names</code>).</p>
<p><img alt="Calibration Multiclass Brier" src="../images/calibration_multiclass_brier.png" title="Calibration Multiclass Brier" /></p>
<h3 id="class-frequency-vs-f1-score">Class Frequency vs. F1 score<a class="headerlink" href="#class-frequency-vs-f1-score" title="Permanent link">&para;</a></h3>
<h4 id="frequency_vs_f1">frequency_vs_f1<a class="headerlink" href="#frequency_vs_f1" title="Permanent link">&para;</a></h4>
<p>This visualization uses the <code>ground_truth_metadata</code>, <code>field</code>, <code>prediction_statistics</code> and <code>model_names</code> parameters.
<code>field</code> needs to be a category.
For each model (in the aligned lists of <code>prediction_statistics</code> and <code>model_names</code>), produces two plots statistics of predictions for the specified <code>field</code>.</p>
<p>The first plot is a line plot with one x axis representing the different classes and two vertical axes colored in orange and blue respectively.
The orange one is the frequency of the class and an orange line is plotted to show the trend.
The blue one is the F1 score for that class and a blue line is plotted to show the trend. 
The classes on the x axis are sorted by f1 score.</p>
<p><img alt="Frequency vs F1 sorted by F1" src="../images/freq_vs_f1_sorted_f1.png" title="Frequency vs F1 sorted by F1" /></p>
<p>The second plot has the same structure of the first one, but the axes are flipped and the classes on the x axis are sorted by frequency.</p>
<p><img alt="Frequency vs F1 sorted by Frequency" src="../images/freq_vs_f1_sorted_freq.png" title="Frequency vs F1 sorted by Frequency" /></p>
                
                  
                
              
              
                


              
            </article>
          </div>
        </div>
      </main>
      
        

<!-- Application footer -->
<footer class="md-footer">

    <!-- Link to previous and/or next page -->
    
    <div class="md-footer-nav">
        <nav class="md-footer-nav__inner md-grid">

            <!-- Link to previous page -->
            
            <a class="md-flex md-footer-nav__link md-footer-nav__link--prev"
               href="../examples/"
               rel="prev"
               title="Examples">
                <div class="md-flex__cell md-flex__cell--shrink">
                    <i class="md-icon md-icon--arrow-back
                    md-footer-nav__button"></i>
                </div>
                <div class="md-flex__cell md-flex__cell--stretch
                  md-footer-nav__title">
              <span class="md-flex__ellipsis">
                <span class="md-footer-nav__direction">
                  Previous
                </span>
                Examples
              </span>
                </div>
            </a>
            

            <!-- Link to next page -->
            
            <a class="md-flex md-footer-nav__link md-footer-nav__link--next"
               href="../developer_guide/"
               rel="next"
               title="Developer Guide">
                <div class="md-flex__cell md-flex__cell--stretch
                  md-footer-nav__title">
              <span class="md-flex__ellipsis">
                <span class="md-footer-nav__direction">
                  Next
                </span>
                Developer Guide
              </span>
                </div>
                <div class="md-flex__cell md-flex__cell--shrink">
                    <i class="md-icon md-icon--arrow-forward
                    md-footer-nav__button"></i>
                </div>
            </a>
            
        </nav>
    </div>
    

    <!-- Further information -->
    <div class="md-footer-meta md-typeset">
        <div class="md-footer-meta__inner md-grid">

            <!-- Copyright and theme information -->
            <div class="md-footer-copyright">
                <div class="footer-logo-smallpad"></div>
                
                <div class="md-footer-copyright__highlight">
                    Copyright &copy; 2018 - 2019 Uber Technologies Inc.
                </div>
                
                Website by <a href="http://w4nderlu.st">w4nderlust</a> powered by
                <a href="https://www.mkdocs.org">MkDocs</a>,
                <a href="https://squidfunk.github.io/mkdocs-material/">Material for MkDocs</a>,
                <a href="http://www.styleshout.com/">styleshout</a> and
                <a href="http://cables.gl/">cables</a>.
            </div>

            <!-- Social links -->
            
            
            
        </div>
    </div>
</footer>
      
    </div>

    <script src="../assets/javascripts/application.a353778b.js"></script>
      
      <script>app.initialize({version:"1.0.4",url:{base:".."}})</script>


    </body>
</html>